client_example.py
9.75 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
"""
MinerU Tianshu - Client Example
天枢客户端示例
演示如何使用 Python 客户端提交任务和查询状态
"""
import asyncio
import aiohttp
from pathlib import Path
from loguru import logger
import time
from typing import Dict
class TianshuClient:
"""天枢客户端"""
def __init__(self, api_url='http://localhost:8000'):
self.api_url = api_url
self.base_url = f"{api_url}/api/v1"
async def submit_task(
self,
session: aiohttp.ClientSession,
file_path: str,
backend: str = 'pipeline',
lang: str = 'ch',
method: str = 'auto',
formula_enable: bool = True,
table_enable: bool = True,
priority: int = 0
) -> Dict:
"""
提交任务
Args:
session: aiohttp session
file_path: 文件路径
backend: 处理后端
lang: 语言
method: 解析方法
formula_enable: 是否启用公式识别
table_enable: 是否启用表格识别
priority: 优先级
Returns:
响应字典,包含 task_id
"""
with open(file_path, 'rb') as f:
data = aiohttp.FormData()
data.add_field('file', f, filename=Path(file_path).name)
data.add_field('backend', backend)
data.add_field('lang', lang)
data.add_field('method', method)
data.add_field('formula_enable', str(formula_enable).lower())
data.add_field('table_enable', str(table_enable).lower())
data.add_field('priority', str(priority))
async with session.post(f'{self.base_url}/tasks/submit', data=data) as resp:
if resp.status == 200:
result = await resp.json()
logger.info(f"✅ Submitted: {file_path} -> Task ID: {result['task_id']}")
return result
else:
error = await resp.text()
logger.error(f"❌ Failed to submit {file_path}: {error}")
return {'success': False, 'error': error}
async def get_task_status(self, session: aiohttp.ClientSession, task_id: str) -> Dict:
"""
查询任务状态
Args:
session: aiohttp session
task_id: 任务ID
Returns:
任务状态字典
"""
async with session.get(f'{self.base_url}/tasks/{task_id}') as resp:
if resp.status == 200:
return await resp.json()
else:
return {'success': False, 'error': 'Task not found'}
async def wait_for_task(
self,
session: aiohttp.ClientSession,
task_id: str,
timeout: int = 600,
poll_interval: int = 2
) -> Dict:
"""
等待任务完成
Args:
session: aiohttp session
task_id: 任务ID
timeout: 超时时间(秒)
poll_interval: 轮询间隔(秒)
Returns:
最终任务状态
"""
start_time = time.time()
while True:
status = await self.get_task_status(session, task_id)
if not status.get('success'):
logger.error(f"❌ Failed to get status for task {task_id}")
return status
task_status = status.get('status')
if task_status == 'completed':
logger.info(f"✅ Task {task_id} completed!")
logger.info(f" Output: {status.get('result_path')}")
return status
elif task_status == 'failed':
logger.error(f"❌ Task {task_id} failed!")
logger.error(f" Error: {status.get('error_message')}")
return status
elif task_status == 'cancelled':
logger.warning(f"⚠️ Task {task_id} was cancelled")
return status
# 检查超时
if time.time() - start_time > timeout:
logger.error(f"⏱️ Task {task_id} timeout after {timeout}s")
return {'success': False, 'error': 'timeout'}
# 等待后继续轮询
await asyncio.sleep(poll_interval)
async def get_queue_stats(self, session: aiohttp.ClientSession) -> Dict:
"""获取队列统计"""
async with session.get(f'{self.base_url}/queue/stats') as resp:
return await resp.json()
async def cancel_task(self, session: aiohttp.ClientSession, task_id: str) -> Dict:
"""取消任务"""
async with session.delete(f'{self.base_url}/tasks/{task_id}') as resp:
return await resp.json()
async def example_single_task():
"""示例1:提交单个任务并等待完成"""
logger.info("=" * 60)
logger.info("示例1:提交单个任务")
logger.info("=" * 60)
client = TianshuClient()
async with aiohttp.ClientSession() as session:
# 提交任务
result = await client.submit_task(
session,
file_path='../../demo/pdfs/demo1.pdf',
backend='pipeline',
lang='ch',
formula_enable=True,
table_enable=True
)
if result.get('success'):
task_id = result['task_id']
# 等待完成
logger.info(f"⏳ Waiting for task {task_id} to complete...")
final_status = await client.wait_for_task(session, task_id)
return final_status
async def example_batch_tasks():
"""示例2:批量提交多个任务并并发等待"""
logger.info("=" * 60)
logger.info("示例2:批量提交多个任务")
logger.info("=" * 60)
client = TianshuClient()
# 准备任务列表
files = [
'../../demo/pdfs/demo1.pdf',
'../../demo/pdfs/demo2.pdf',
'../../demo/pdfs/demo3.pdf',
]
async with aiohttp.ClientSession() as session:
# 并发提交所有任务
logger.info(f"📤 Submitting {len(files)} tasks...")
submit_tasks = [
client.submit_task(session, file)
for file in files
]
results = await asyncio.gather(*submit_tasks)
# 提取 task_ids
task_ids = [r['task_id'] for r in results if r.get('success')]
logger.info(f"✅ Submitted {len(task_ids)} tasks successfully")
# 并发等待所有任务完成
logger.info(f"⏳ Waiting for all tasks to complete...")
wait_tasks = [
client.wait_for_task(session, task_id)
for task_id in task_ids
]
final_results = await asyncio.gather(*wait_tasks)
# 统计结果
completed = sum(1 for r in final_results if r.get('status') == 'completed')
failed = sum(1 for r in final_results if r.get('status') == 'failed')
logger.info("=" * 60)
logger.info(f"📊 Results: {completed} completed, {failed} failed")
logger.info("=" * 60)
return final_results
async def example_priority_tasks():
"""示例3:使用优先级队列"""
logger.info("=" * 60)
logger.info("示例3:优先级队列")
logger.info("=" * 60)
client = TianshuClient()
async with aiohttp.ClientSession() as session:
# 提交低优先级任务
low_priority = await client.submit_task(
session,
file_path='../../demo/pdfs/demo1.pdf',
priority=0
)
logger.info(f"📝 Low priority task: {low_priority['task_id']}")
# 提交高优先级任务
high_priority = await client.submit_task(
session,
file_path='../../demo/pdfs/demo2.pdf',
priority=10
)
logger.info(f"🔥 High priority task: {high_priority['task_id']}")
# 高优先级任务会先被处理
logger.info("⏳ 高优先级任务将优先处理...")
async def example_queue_monitoring():
"""示例4:监控队列状态"""
logger.info("=" * 60)
logger.info("示例4:监控队列状态")
logger.info("=" * 60)
client = TianshuClient()
async with aiohttp.ClientSession() as session:
# 获取队列统计
stats = await client.get_queue_stats(session)
logger.info("📊 Queue Statistics:")
logger.info(f" Total: {stats.get('total', 0)}")
for status, count in stats.get('stats', {}).items():
logger.info(f" {status:12s}: {count}")
async def main():
"""主函数"""
import sys
if len(sys.argv) > 1:
example = sys.argv[1]
else:
example = 'all'
try:
if example == 'single' or example == 'all':
await example_single_task()
print()
if example == 'batch' or example == 'all':
await example_batch_tasks()
print()
if example == 'priority' or example == 'all':
await example_priority_tasks()
print()
if example == 'monitor' or example == 'all':
await example_queue_monitoring()
print()
except Exception as e:
logger.error(f"Example failed: {e}")
import traceback
traceback.print_exc()
if __name__ == '__main__':
"""
使用方法:
# 运行所有示例
python client_example.py
# 运行特定示例
python client_example.py single
python client_example.py batch
python client_example.py priority
python client_example.py monitor
"""
asyncio.run(main())