Storage.py

# encoding: utf-8

''' 以bsddb-cpickle为引擎, key-value的形式存储、检索数据
'''

import bsddb
import cPickle as pickle

class Storage:

    def __init__(self, storagefile='storage.db'):
        self.store = bsddb.hashopen(storagefile, 'c')

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.store.close()

    def __getitem__(self, url):
        content = self.store.get(url)
        return None if not content else pickle.loads(content)

    def __setitem__(self, url, content):
        self.store[url] = pickle.dumps(content)

    def keys(self):
        return self.store.keys()
    
    def values(self):
        return self.store.values()
    
    def items(self):
        return self.store.items()


def tester():
    kache = Storage()
    with kache:
        kache['a'] = 'tester'
        kache['b'] = 1234
        kache['c'] = [12.5, 'Tester', 32,]
        
        for r in 'abcd':
            print kache[r]

if __name__ == '__main__':
    tester()

FileNotFoundError: File not found 2025-07-24 05:02:17.666 INFO [Dummy-2] [document_indexing_task.py:80] - Processed dataset: 695f9f4f-da54-44f9-a5f5-5cbef1634796 latency: 0.6217305730096996 2025-07-24 05:02:17.874 INFO [Dummy-2] [trace.py:128] - Task tasks.document_indexing_task.document_indexing_task[93a86dcd-ce2c-4535-9465-fab22b79b9c7] succeeded in 0.8317920980043709s: None 2025-07-24 05:09:50.999 INFO [MainThread] [strategy.py:161] - Task tasks.retry_document_indexing_task.retry_document_indexing_task[8bd046ef-ba37-4ca4-a5ac-bd90b65763ac] received 2025-07-24 05:09:51.037 INFO [Dummy-3] [retry_document_indexing_task.py:60] - Start retry document: 21663e49-588c-4669-9338-074c46b9dea7 2025-07-24 05:09:51.112 ERROR [Dummy-3] [indexing_runner.py:96] - consume document failed Traceback (most recent call last): File "/app/api/core/indexing_runner.py", line 70, in run text_docs = self._extract(index_processor, dataset_document, processing_rule.to_dict()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/indexing_runner.py", line 356, in _extract text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"]) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/rag/index_processor/processor/paragraph_index_processor.py", line 22, in extract text_docs = ExtractProcessor.extract( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/rag/extractor/extract_processor.py", line 103, in extract storage.download(upload_file.key, file_path) File "/app/api/extensions/ext_storage.py", line 97, in download self.storage_runner.download(filename, target_filepath) File "/app/api/extensions/storage/opendal_storage.py", line 67, in download raise FileNotFoundError("File not found") FileNotFoundError: File not found 2025-07-24 05:09:51.134 INFO [Dummy-3] [retry_document_indexing_task.py:102] - Retry dataset: 695f9f4f-da54-44f9-a5f5-5cbef1634796 latency: 0.12441904097795486 2025-07-24 05:09:51.379 INFO [Dummy-3] [trace.py:128] - Task tasks.retry_document_indexing_task.retry_document_indexing_task[8bd046ef-ba37-4ca4-a5ac-bd90b65763ac] succeeded in 0.3706196919083595s: None 2025-07-24 05:10:08.712 INFO [MainThread] [strategy.py:161] - Task tasks.document_indexing_task.document_indexing_task[d5b949ad-ba43-4e84-8160-80aa9a38d106] received 2025-07-24 05:10:08.721 INFO [Dummy-4] [document_indexing_task.py:63] - Start process document: 7c3d3436-7164-41dc-afc6-336b0ef53969 2025-07-24 05:10:08.742 ERROR [Dummy-4] [indexing_runner.py:96] - consume document failed Traceback (most recent call last): File "/app/api/core/indexing_runner.py", line 70, in run text_docs = self._extract(index_processor, dataset_document, processing_rule.to_dict()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/indexing_runner.py", line 356, in _extract text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"]) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/rag/index_processor/processor/paragraph_index_processor.py", line 22, in extract text_docs = ExtractProcessor.extract( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/rag/extractor/extract_processor.py", line 103, in extract storage.download(upload_file.key, file_path) File "/app/api/extensions/ext_storage.py", line 97, in download self.storage_runner.download(filename, target_filepath) File "/app/api/extensions/storage/opendal_storage.py", line 67, in download raise FileNotFoundError("File not found") FileNotFoundError: File not found 2025-07-24 05:10:08.748 INFO [Dummy-4] [document_indexing_task.py:80] - Processed dataset: c6c02118-21ea-4f2b-9770-de6d41cb4e0a latency: 0.03286655503325164 2025-07-24 05:10:08.907 INFO [Dummy-4] [trace.py:128] - Task tasks.document_indexing_task.document_indexing_task[d5b949ad-ba43-4e84-8160-80aa9a38d106] succeeded in 0.1924347539898008s: None 2025-07-24 05:17:23.919 INFO [MainThread] [strategy.py:161] - Task tasks.document_indexing_task.document_indexing_task[c95b3b5f-79ec-4591-8a70-7eeb0be6f4f2] received 2025-07-24 05:17:23.957 INFO [Dummy-5] [document_indexing_task.py:63] - Start process document: ddf5b24a-6c5d-47b3-9f23-6fb6eb884f0c 2025-07-24 05:17:23.995 ERROR [Dummy-5] [indexing_runner.py:96] - consume document failed Traceback (most recent call last): File "/app/api/core/indexing_runner.py", line 70, in run text_docs = self._extract(index_processor, dataset_document, processing_rule.to_dict()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/indexing_runner.py", line 356, in _extract text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"]) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/rag/index_processor/processor/paragraph_index_processor.py", line 22, in extract text_docs = ExtractProcessor.extract( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/rag/extractor/extract_processor.py", line 103, in extract storage.download(upload_file.key, file_path) File "/app/api/extensions/ext_storage.py", line 97, in download self.storage_runner.download(filename, target_filepath) File "/app/api/extensions/storage/opendal_storage.py", line 67, in download raise FileNotFoundError("File not found") FileNotFoundError: File not found 2025-07-24 05:17:24.008 INFO [Dummy-5] [document_indexing_task.py:80] - Processed dataset: 8af61382-18c0-4651-bb1e-b17e21876af7 latency: 0.08017416414804757 2025-07-24 05:17:24.214 INFO [Dummy-5] [trace.py:128] - Task tasks.document_indexing_task.document_indexing_task[c95b3b5f-79ec-4591-8a70-7eeb0be6f4f2] succeeded in 0.28705954691395164s: None 2025-07-24 05:19:08.234 INFO [MainThread] [strategy.py:161] - Task tasks.retry_document_indexing_task.retry_document_indexing_task[b4ba321d-f402-4be3-91a4-0f2eb8df9b1e] received 2025-07-24 05:19:08.251 INFO [Dummy-6] [retry_document_indexing_task.py:60] - Start retry document: ddf5b24a-6c5d-47b3-9f23-6fb6eb884f0c 2025-07-24 05:19:08.287 ERROR [Dummy-6] [indexing_runner.py:96] - consume document failed Traceback (most recent call last): File "/app/api/core/indexing_runner.py", line 70, in run text_docs = self._extract(index_processor, dataset_document, processing_rule.to_dict()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/indexing_runner.py", line 356, in _extract text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"]) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/rag/index_processor/processor/paragraph_index_processor.py", line 22, in extract text_docs = ExtractProcessor.extract( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/api/core/rag/extractor/extract_processor.py", line 103, in extract storage.download(upload_file.key, file_path) File "/app/api/extensions/ext_storage.py", line 97, in download self.storage_runner.download(filename, target_filepath) File "/app/api/extensions/storage/opendal_storage.py", line 67, in download raise FileNotFoundError("File not found") FileNotFoundError: File not found 2025-07-24 05:19:08.298 INFO [Dummy-6] [retry_document_indexing_task.py:102] - Retry dataset: 8af61382-18c0-4651-bb1e-b17e21876af7 latency: 0.06023550289683044 2025-07-24 05:19:08.525 INFO [Dummy-6] [trace.py:128] - Task tasks.retry_document_indexing_task.retry_document_indexing_task[b4ba321d-f402-4be3-91a4-0f2eb8df9b1e] succeeded in 0.28708126000128686s: None [root@clickhouse-master web]#
最新发布
07-25
Stack (most recent call first): File "F:\QYY3.5\sd-webui-forge-aki-v1.0\python\Lib\site-packages\torch\storage.py", line 469 in __getitem__ File "F:\QYY3.5\sd-webui-forge-aki-v1.0\python\Lib\site-packages\safetensors\torch.py", line 310 in load_file File "F:\QYY3.5\sd-webui-forge-aki-v1.0\modules_forge\patch_basic.py", line 38 in loader File "F:\QYY3.5\sd-webui-forge-aki-v1.0\backend\utils.py", line 26 in load_torch_file File "F:\QYY3.5\sd-webui-forge-aki-v1.0\backend\loader.py", line 238 in split_state_dict File "F:\QYY3.5\sd-webui-forge-aki-v1.0\backend\loader.py", line 274 in forge_loader File "F:\QYY3.5\sd-webui-forge-aki-v1.0\python\Lib\site-packages\torch\utils\_contextlib.py", line 116 in decorate_context File "F:\QYY3.5\sd-webui-forge-aki-v1.0\modules\sd_models.py", line 504 in forge_model_reload File "F:\QYY3.5\sd-webui-forge-aki-v1.0\python\Lib\site-packages\torch\utils\_contextlib.py", line 116 in decorate_context File "F:\QYY3.5\sd-webui-forge-aki-v1.0\modules\processing.py", line 804 in manage_model_and_prompt_cache File "F:\QYY3.5\sd-webui-forge-aki-v1.0\modules\processing.py", line 836 in process_images File "F:\QYY3.5\sd-webui-forge-aki-v1.0\modules\txt2img.py", line 131 in txt2img_function File "F:\QYY3.5\sd-webui-forge-aki-v1.0\modules_forge\main_thread.py", line 30 in work File "F:\QYY3.5\sd-webui-forge-aki-v1.0\modules_forge\main_thread.py", line 48 in loop File "F:\QYY3.5\sd-webui-forge-aki-v1.0\modules\launch_utils.py", line 557 in start File "F:\QYY3.5\sd-webui-forge-aki-v1.0\launch.py", line 50 in main File "F:\QYY3.5\sd-webui-forge-aki-v1.0\launch.py", line 54 in <module>
03-22
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值