Leon/docutranslate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化minerU的超时设置

Browse Source
This commit is contained in:
xunbu
2025-05-24 22:15:31 +08:00
parent 3c92d8d25f
commit d5712862b2
2 changed files with 68 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
.idea/workspace.xml generated
View File

@@ -7,7 +7,6 @@
<list default="true" id="6b18b44a-df57-4212-a857-9e291ebe5dd2" name="更改" comment=""> <list default="true" id="6b18b44a-df57-4212-a857-9e291ebe5dd2" name="更改" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/docutranslate/converter/converter_mineru.py" beforeDir="false" afterPath="$PROJECT_DIR$/docutranslate/converter/converter_mineru.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/docutranslate/converter/converter_mineru.py" beforeDir="false" afterPath="$PROJECT_DIR$/docutranslate/converter/converter_mineru.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pyproject.toml" beforeDir="false" afterPath="$PROJECT_DIR$/pyproject.toml" afterDir="false" />
</list> </list>
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" /> <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -37,62 +36,62 @@
<option name="hideEmptyMiddlePackages" value="true" /> <option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" /> <option name="showLibraryContents" value="true" />
</component> </component>
<component name="PropertiesComponent">{ <component name="PropertiesComponent"><![CDATA[{
&quot;keyToString&quot;: { "keyToString": {
&quot;DefaultHtmlFileTemplate&quot;: &quot;HTML File&quot;, "DefaultHtmlFileTemplate": "HTML File",
&quot;JavaScript 调试.output.html (1).executor&quot;: &quot;Run&quot;, "JavaScript 调试.output.html (1).executor": "Run",
&quot;JavaScript 调试.output.html.executor&quot;: &quot;Run&quot;, "JavaScript 调试.output.html.executor": "Run",
&quot;JavaScript 调试.regex.md_中文.html.executor&quot;: &quot;Run&quot;, "JavaScript 调试.regex.md_中文.html.executor": "Run",
&quot;JavaScript 调试.regex_中文.html.executor&quot;: &quot;Run&quot;, "JavaScript 调试.regex_中文.html.executor": "Run",
&quot;JavaScript 调试.test.html.executor&quot;: &quot;Run&quot;, "JavaScript 调试.test.html.executor": "Run",
&quot;JavaScript 调试.test2.html.executor&quot;: &quot;Run&quot;, "JavaScript 调试.test2.html.executor": "Run",
&quot;JavaScript 调试.test2_英文.html.executor&quot;: &quot;Run&quot;, "JavaScript 调试.test2_英文.html.executor": "Run",
&quot;JavaScript 调试.test4-1_中文.html.executor&quot;: &quot;Run&quot;, "JavaScript 调试.test4-1_中文.html.executor": "Run",
&quot;JavaScript 调试.互联网认证授权机制.html.executor&quot;: &quot;Run&quot;, "JavaScript 调试.互联网认证授权机制.html.executor": "Run",
&quot;JavaScript 调试.互联网认证授权机制_英文.html.executor&quot;: &quot;Run&quot;, "JavaScript 调试.互联网认证授权机制_英文.html.executor": "Run",
&quot;JavaScript 调试.毕业论文_英文.html.executor&quot;: &quot;Run&quot;, "JavaScript 调试.毕业论文_英文.html.executor": "Run",
&quot;ModuleVcsDetector.initialDetectionPerformed&quot;: &quot;true&quot;, "ModuleVcsDetector.initialDetectionPerformed": "true",
&quot;Python 测试.Python 测试 (markdown_mask.py 内).executor&quot;: &quot;Run&quot;, "Python 测试.Python 测试 (markdown_mask.py 内).executor": "Run",
&quot;Python 测试.markdown_mask.Test.test_basic_link_masking 的 Python 测试.executor&quot;: &quot;Run&quot;, "Python 测试.markdown_mask.Test.test_basic_link_masking 的 Python 测试.executor": "Run",
&quot;Python 测试.pytest (test_html.py 内).executor&quot;: &quot;Run&quot;, "Python 测试.pytest (test_html.py 内).executor": "Run",
&quot;Python.1test.executor&quot;: &quot;Run&quot;, "Python.1test.executor": "Run",
&quot;Python.2test2 (1).executor&quot;: &quot;Run&quot;, "Python.2test2 (1).executor": "Run",
&quot;Python.PDFtranslater (1).executor&quot;: &quot;Run&quot;, "Python.PDFtranslater (1).executor": "Run",
&quot;Python.PDFtranslater (2).executor&quot;: &quot;Run&quot;, "Python.PDFtranslater (2).executor": "Run",
&quot;Python.agent.executor&quot;: &quot;Debug&quot;, "Python.agent.executor": "Debug",
&quot;Python.agent_utils.executor&quot;: &quot;Run&quot;, "Python.agent_utils.executor": "Run",
&quot;Python.app (1).executor&quot;: &quot;Run&quot;, "Python.app (1).executor": "Run",
&quot;Python.app.executor&quot;: &quot;Run&quot;, "Python.app.executor": "Run",
&quot;Python.app2.executor&quot;: &quot;Run&quot;, "Python.app2.executor": "Run",
&quot;Python.app_test (1).executor&quot;: &quot;Run&quot;, "Python.app_test (1).executor": "Run",
&quot;Python.convert.executor&quot;: &quot;Run&quot;, "Python.convert.executor": "Run",
&quot;Python.converter_docling.executor&quot;: &quot;Run&quot;, "Python.converter_docling.executor": "Run",
&quot;Python.converter_mineru.executor&quot;: &quot;Run&quot;, "Python.converter_mineru.executor": "Run",
&quot;Python.markdown_splitter.executor&quot;: &quot;Debug&quot;, "Python.markdown_splitter.executor": "Debug",
&quot;Python.markdown_utils.executor&quot;: &quot;Run&quot;, "Python.markdown_utils.executor": "Run",
&quot;Python.test.executor&quot;: &quot;Run&quot;, "Python.test.executor": "Run",
&quot;Python.test1.executor&quot;: &quot;Run&quot;, "Python.test1.executor": "Run",
&quot;Python.test2.executor&quot;: &quot;Run&quot;, "Python.test2.executor": "Run",
&quot;Python.test3.executor&quot;: &quot;Run&quot;, "Python.test3.executor": "Run",
&quot;Python.test4.executor&quot;: &quot;Run&quot;, "Python.test4.executor": "Run",
&quot;Python.testhtml.executor&quot;: &quot;Run&quot;, "Python.testhtml.executor": "Run",
&quot;Python.translater.executor&quot;: &quot;Run&quot;, "Python.translater.executor": "Run",
&quot;Python.切分测试.executor&quot;: &quot;Run&quot;, "Python.切分测试.executor": "Run",
&quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;, "RunOnceActivity.ShowReadmeOnStart": "true",
&quot;RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager&quot;: &quot;true&quot;, "RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager": "true",
&quot;RunOnceActivity.git.unshallow&quot;: &quot;true&quot;, "RunOnceActivity.git.unshallow": "true",
&quot;git-widget-placeholder&quot;: &quot;dev&quot;, "git-widget-placeholder": "main",
&quot;last_opened_file_path&quot;: &quot;C:/Users/jxgm/Desktop/translate/docutranslate&quot;, "last_opened_file_path": "C:/Users/jxgm/Desktop/translate/docutranslate",
&quot;list.type.of.created.stylesheet&quot;: &quot;CSS&quot;, "list.type.of.created.stylesheet": "CSS",
&quot;node.js.detected.package.eslint&quot;: &quot;true&quot;, "node.js.detected.package.eslint": "true",
&quot;node.js.detected.package.tslint&quot;: &quot;true&quot;, "node.js.detected.package.tslint": "true",
&quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;, "node.js.selected.package.eslint": "(autodetect)",
&quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;, "node.js.selected.package.tslint": "(autodetect)",
&quot;nodejs_package_manager_path&quot;: &quot;npm&quot;, "nodejs_package_manager_path": "npm",
&quot;settings.editor.selected.configurable&quot;: &quot;preferences.pluginManager&quot;, "settings.editor.selected.configurable": "preferences.pluginManager",
&quot;vue.rearranger.settings.migration&quot;: &quot;true&quot; "vue.rearranger.settings.migration": "true"
} }
}</component> }]]></component>
<component name="RecentsManager"> <component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS"> <key name="CopyFile.RECENT_KEYS">
<recent name="C:\Users\jxgm\Desktop\translate\docutranslate" /> <recent name="C:\Users\jxgm\Desktop\translate\docutranslate" />

18
docutranslate/converter/converter_mineru.py
View File

@@ -8,14 +8,22 @@ from docutranslate.utils.markdown_utils import embed_inline_image_from_zip
URL = 'https://mineru.net/api/v4/file-urls/batch' URL = 'https://mineru.net/api/v4/file-urls/batch'
client = httpx.Client(trust_env=False) timeout = httpx.Timeout(
connect=5.0, # 连接超时 (建立连接的最长时间)
read=120.0, # 读取超时 (等待服务器响应的最长时间)
write=120.0, # 写入超时 (发送数据的最长时间)
pool=1.0 # 从连接池获取连接的超时时间
)
client = httpx.Client(trust_env=False,timeout=timeout)
# TODO: 提供更详细的logger # TODO: 提供更详细的logger
class ConverterMineru(Converter): class ConverterMineru(Converter):
def __init__(self, token: str, formula=True): def __init__(self, token: str, formula=True):
self.mineru_token = token.strip() self.mineru_token = token.strip()
self.client_async = httpx.AsyncClient() self.client_async = httpx.AsyncClient(timeout=timeout)
self.formula = formula self.formula = formula
def _get_header(self): def _get_header(self):
@@ -36,7 +44,7 @@ class ConverterMineru(Converter):
def upload(self, document: Document): def upload(self, document: Document):
# 获取上传链接 # 获取上传链接
response = client.post(URL, headers=self._get_header(), json=self._get_upload_data(document),timeout=120) response = client.post(URL, headers=self._get_header(), json=self._get_upload_data(document))
response.raise_for_status() response.raise_for_status()
result = response.json() result = response.json()
# print('response success. result:{}'.format(result)) # print('response success. result:{}'.format(result))
@@ -45,7 +53,7 @@ class ConverterMineru(Converter):
urls = result["data"]["file_urls"] urls = result["data"]["file_urls"]
# print('batch_id:{},urls:{}'.format(batch_id, urls)) # print('batch_id:{},urls:{}'.format(batch_id, urls))
# 获取 # 获取
res_upload = client.put(urls[0], content=document.filebytes,timeout=120) res_upload = client.put(urls[0], content=document.filebytes)
res_upload.raise_for_status() res_upload.raise_for_status()
# print(f"{urls[0]} upload success") # print(f"{urls[0]} upload success")
return batch_id return batch_id
@@ -109,7 +117,7 @@ def get_md_from_zip_url_with_inline_images(
""" """
try: try:
print(f"正在从 {zip_url} 下载ZIP文件 (使用 httpx.get)...") print(f"正在从 {zip_url} 下载ZIP文件 (使用 httpx.get)...")
response = client.get(zip_url, timeout=120.0) # 增加超时 response = client.get(zip_url) # 增加超时
response.raise_for_status() response.raise_for_status()
print("ZIP文件下载完成。") print("ZIP文件下载完成。")
return embed_inline_image_from_zip(response.content, filename_in_zip=filename_in_zip, encoding=encoding) return embed_inline_image_from_zip(response.content, filename_in_zip=filename_in_zip, encoding=encoding)