chenbhao commited on
Commit
6dc252c
·
1 Parent(s): f7a8a02

refactor(WebFetchTool): remove dead Python webtools fallback

Browse files

WebFetch already had a complete pure-TS implementation (turndown +
domino + native fetch). The Python branch spawned scripts/python_webtools.py
which no longer exists, so it always failed and fell back to the TS path.
Remove the dead code and its .venv dependency.

Files changed (1) hide show
  1. src/tools/WebFetchTool/utils.ts +0 -101
src/tools/WebFetchTool/utils.ts CHANGED
@@ -137,86 +137,6 @@ async function retryWithBackoff<T>(
137
  throw lastError
138
  }
139
 
140
- /**
141
- * Fetch URL content using Python webtools script
142
- * Reference: nanobot's web.py implementation
143
- * Returns markdown formatted content with metadata
144
- * Returns null if should fall back to direct fetch
145
- */
146
- async function fetchWithPythonWebtools(url: string): Promise<{
147
- content: string
148
- contentType: string
149
- title?: string
150
- finalUrl?: string
151
- } | null> {
152
- console.log(`[WebFetch] Fetching via Python webtools: ${url}`)
153
-
154
- try {
155
- const { spawn } = await import('child_process')
156
-
157
- return new Promise((resolve, reject) => {
158
- const pythonScript = process.cwd() + '/scripts/python_webtools.py'
159
- const maxChars = 50000
160
-
161
- const child = spawn('.venv/bin/python', [pythonScript, 'web_fetch', url, String(50000)], {
162
- cwd: process.cwd(),
163
- })
164
-
165
- let stdout = ''
166
- let stderr = ''
167
-
168
- child.stdout.on('data', (data) => {
169
- stdout += data.toString()
170
- })
171
-
172
- child.stderr.on('data', (data) => {
173
- stderr += data.toString()
174
- })
175
-
176
- child.on('close', (code) => {
177
- if (code !== 0) {
178
- console.error('[WebFetch] Python script failed:', stderr)
179
- resolve(null) // Return null to trigger fallback
180
- return
181
- }
182
-
183
- try {
184
- const result = JSON.parse(stdout)
185
-
186
- if (!result.success) {
187
- console.error('[WebFetch] Python fetch failed:', result.error)
188
- resolve(null) // Return null to trigger fallback
189
- return
190
- }
191
-
192
- console.log(`[WebFetch] Python returned ${result.length} bytes`)
193
-
194
- resolve({
195
- content: result.text,
196
- contentType: 'text/markdown',
197
- title: undefined, // Python already includes title in text
198
- finalUrl: result.finalUrl || url,
199
- })
200
- } catch (error) {
201
- console.error('[WebFetch] Failed to parse Python output:', error)
202
- resolve(null) // Return null to trigger fallback
203
- }
204
- })
205
-
206
- child.on('error', (error) => {
207
- console.error('[WebFetch] Failed to start Python process:', error)
208
- resolve(null) // Return null to trigger fallback
209
- })
210
- })
211
- } catch (error) {
212
- console.error('[WebFetch] Failed to call Python webtools:', error)
213
- logError('WebFetch: Failed to call Python webtools', error)
214
- return null // Return null to trigger fallback
215
- }
216
- }
217
-
218
-
219
-
220
  // Cache for storing fetched URL content
221
  type CacheEntry = {
222
  bytes: number
@@ -644,27 +564,6 @@ export async function getURLMarkdownContent(
644
  console.error('[WebFetch] Local fetch failed:', error)
645
  logError('Local fetch failed', error)
646
 
647
- // Try Python webtools fallback
648
- try {
649
- console.log('[WebFetch] Trying Python webtools fallback')
650
- const pythonResult = await fetchWithPythonWebtools(upgradedUrl)
651
-
652
- if (pythonResult) {
653
- const bytes = Buffer.byteLength(pythonResult.content)
654
- const entry: CacheEntry = {
655
- bytes,
656
- code: 200,
657
- codeText: 'OK',
658
- content: pythonResult.content,
659
- contentType: pythonResult.contentType,
660
- }
661
- URL_CACHE.set(url, entry, { size: Math.max(1, bytes) })
662
- return entry
663
- }
664
- } catch (pythonError) {
665
- console.error('[WebFetch] Python fallback also failed:', pythonError)
666
- }
667
-
668
  throw new Error(`Failed to fetch URL: ${error instanceof Error ? error.message : String(error)}`)
669
  }
670
  }
 
137
  throw lastError
138
  }
139
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
140
  // Cache for storing fetched URL content
141
  type CacheEntry = {
142
  bytes: number
 
564
  console.error('[WebFetch] Local fetch failed:', error)
565
  logError('Local fetch failed', error)
566
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
567
  throw new Error(`Failed to fetch URL: ${error instanceof Error ? error.message : String(error)}`)
568
  }
569
  }