From 574f188f9caeb1fbc321c539fb662c41c4268fd3 Mon Sep 17 00:00:00 2001 From: zwfxxxxx Date: Thu, 15 Oct 2026 14:34:31 +0800 Subject: [PATCH] remove legacy big data screen plugin IO tasks --- dtable_events/dtable_io/__init__.py | 36 +---- dtable_events/dtable_io/request_handler.py | 53 -------- dtable_events/dtable_io/task_manager.py | 22 --- dtable_events/dtable_io/utils.py | 150 --------------------- 4 files changed, 1 insertion(+), 260 deletions(-) diff --git a/dtable_events/dtable_io/__init__.py b/dtable_events/dtable_io/__init__.py index 4a2c3cf7..6f52ee24 100644 --- a/dtable_events/dtable_io/__init__.py +++ b/dtable_events/dtable_io/__init__.py @@ -23,7 +23,7 @@ prepare_dtable_json_from_memory, update_page_design_static_image, \ copy_src_auto_rules_to_json, create_auto_rules_from_src_dtable, sync_app_users_to_table, \ copy_src_workflows_to_json, create_workflows_from_src_dtable, copy_src_external_app_to_json,\ - create_external_apps_from_src_dtable, zip_big_data_screen, post_big_data_screen_zip_file, \ + create_external_apps_from_src_dtable, \ export_page_design_dir_to_path, update_page_design_content_to_path, upload_page_design, \ download_page_design_file, zip_big_data_screen_app, prepare_asset_files_download, copy_src_archive_backup from dtable_events.db import init_db_session_class @@ -515,40 +515,6 @@ def get_dtable_export_asset_files(username, repo_id, dtable_uuid, files, task_id finally: db_session.close() -def get_dtable_export_big_data_screen(username, repo_id, dtable_uuid, page_id, task_id): - """ - parse json file in big data screen, and zip it for download - """ - tmp_file_path = os.path.join('/tmp/dtable-io', dtable_uuid, 'big-data-screen', str(task_id)) - tmp_zip_path = os.path.join('/tmp/dtable-io', dtable_uuid, 'big-data-screen', str(task_id) + '.zip') - clear_tmp_files_and_dirs(tmp_file_path, tmp_zip_path) - os.makedirs(tmp_file_path.rstrip('/') + '/images', exist_ok=True) - - try: - zip_big_data_screen(username, repo_id, dtable_uuid, page_id, tmp_file_path) - shutil.make_archive(tmp_zip_path.split('.')[0], 'zip', root_dir=tmp_file_path) - except Exception as e: - dtable_io_logger.error('export big data screen from dtable failed. ERROR: {}'.format(e)) - else: - dtable_io_logger.info('export big data screen from dtable: %s success!', dtable_uuid) - -def import_big_data_screen(username, repo_id, dtable_uuid, page_id): - """ - parse the zip in tmp folders and upload it - """ - tmp_extracted_path = os.path.join('/tmp/dtable-io', dtable_uuid, 'big_data_screen_zip_extracted/') - try: - post_big_data_screen_zip_file(username, repo_id, dtable_uuid, page_id, tmp_extracted_path) - except Exception as e: - dtable_io_logger.error('import big data screen from dtable failed. ERROR: {}'.format(e)) - else: - dtable_io_logger.info('import big data screen to dtable: %s success!', dtable_uuid) - try: - shutil.rmtree(tmp_extracted_path) - except Exception as e: - dtable_io_logger.error('rm extracted tmp file failed. ERROR: {}'.format(e)) - - def get_dtable_export_big_data_screen_app(username, repo_id, dtable_uuid, app_uuid, app_id, task_id): """ parse json file in big data screen, and zip it for download diff --git a/dtable_events/dtable_io/request_handler.py b/dtable_events/dtable_io/request_handler.py index 88420a90..30e10f51 100644 --- a/dtable_events/dtable_io/request_handler.py +++ b/dtable_events/dtable_io/request_handler.py @@ -118,59 +118,6 @@ def add_import_task(): return make_response(({'task_id': task_id}, 200)) -@app.route('/add-big-data-screen-export-task', methods=['GET']) -def add_big_data_screen_export_task(): - is_valid, error = check_auth_token(request) - if not is_valid: - return make_response((error, 403)) - - if task_manager.tasks_queue.full(): - dtable_io_logger.warning('dtable io server busy, queue size: %d, current tasks: %s, threads is_alive: %s' - % (task_manager.tasks_queue.qsize(), task_manager.current_task_info, - task_manager.threads_is_alive())) - return make_response(('dtable io server busy.', 400)) - - username = request.args.get('username') - repo_id = request.args.get('repo_id') - dtable_uuid = request.args.get('dtable_uuid') - page_id = request.args.get('page_id') - - try: - task_id = task_manager.add_export_dtable_big_data_screen_task( - username, repo_id, dtable_uuid, page_id) - except Exception as e: - dtable_io_logger.error(e) - return make_response((e, 500)) - - return make_response(({'task_id': task_id}, 200)) - -@app.route('/add-big-data-screen-import-task', methods=['GET']) -def add_big_data_screen_import_task(): - is_valid, error = check_auth_token(request) - if not is_valid: - return make_response((error, 403)) - - if task_manager.tasks_queue.full(): - dtable_io_logger.warning('dtable io server busy, queue size: %d, current tasks: %s, threads is_alive: %s' - % (task_manager.tasks_queue.qsize(), task_manager.current_task_info, - task_manager.threads_is_alive())) - return make_response(('dtable io server busy.', 400)) - - username = request.args.get('username') - repo_id = request.args.get('repo_id') - dtable_uuid = request.args.get('dtable_uuid') - page_id = request.args.get('page_id') - - try: - task_id = task_manager.add_import_dtable_big_data_screen_task( - username, repo_id, dtable_uuid, page_id) - except Exception as e: - dtable_io_logger.error(e) - return make_response((e, 500)) - - return make_response(({'task_id': task_id}, 200)) - - @app.route('/add-big-data-screen-app-export-task', methods=['GET']) def add_big_data_screen_app_export_task(): is_valid, error = check_auth_token(request) diff --git a/dtable_events/dtable_io/task_manager.py b/dtable_events/dtable_io/task_manager.py index 296c5222..0b79ed74 100644 --- a/dtable_events/dtable_io/task_manager.py +++ b/dtable_events/dtable_io/task_manager.py @@ -117,28 +117,6 @@ def add_export_dtable_asset_files_task(self, username, repo_id, dtable_uuid, fil publish_metric(self.tasks_queue.qsize(), 'io_task_queue_size', metric_help=TASK_MANAGER_METRIC_HELP) return task_id - @log_function_call - def add_export_dtable_big_data_screen_task(self, username, repo_id, dtable_uuid, page_id): - from dtable_events.dtable_io import get_dtable_export_big_data_screen - task_id = str(uuid.uuid4()) - task = (get_dtable_export_big_data_screen, - (username, repo_id, dtable_uuid, page_id, task_id)) - self.tasks_queue.put(task_id) - self.tasks_map[task_id] = task - publish_metric(self.tasks_queue.qsize(), 'io_task_queue_size', metric_help=TASK_MANAGER_METRIC_HELP) - return task_id - - @log_function_call - def add_import_dtable_big_data_screen_task(self, username, repo_id, dtable_uuid, page_id): - from dtable_events.dtable_io import import_big_data_screen - task_id = str(uuid.uuid4()) - task = (import_big_data_screen, - (username, repo_id, dtable_uuid, page_id)) - self.tasks_queue.put(task_id) - self.tasks_map[task_id] = task - publish_metric(self.tasks_queue.qsize(), 'io_task_queue_size', metric_help=TASK_MANAGER_METRIC_HELP) - return task_id - @log_function_call def add_export_dtable_big_data_screen_app_task(self, username, repo_id, dtable_uuid, app_uuid, app_id): from dtable_events.dtable_io import get_dtable_export_big_data_screen_app diff --git a/dtable_events/dtable_io/utils.py b/dtable_events/dtable_io/utils.py index af9482fd..b39407f5 100644 --- a/dtable_events/dtable_io/utils.py +++ b/dtable_events/dtable_io/utils.py @@ -1622,156 +1622,6 @@ def height_transfer(base_row_height='default'): return round((32 * row_height_mul * 14.4 ) / 24, 2) -def zip_big_data_screen(username, repo_id, dtable_uuid, page_id, tmp_file_path): - from dtable_events.utils import uuid_str_to_36_chars, normalize_file_path, gen_file_get_url - base_dir = '/asset/' + dtable_uuid - big_data_file_path = 'files/plugins/big-data-screen/%(page_id)s/%(page_id)s.json' % ({ - 'page_id': page_id, - }) - big_data_poster_path = 'files/plugins/big-data-screen/%(page_id)s/%(page_id)s.png' % ({ - 'page_id': page_id - }) - - asset_path = "%s/%s" % (base_dir, big_data_file_path) - - poster_asset_path = "%s/%s" % (base_dir, big_data_poster_path) - - # 1. get the json file and poster of big-data-screen page - # a. json file - asset_id = seafile_api.get_file_id_by_path(repo_id, asset_path) - token = seafile_api.get_fileserver_access_token( - repo_id, asset_id, 'view', '', use_onetime=False - ) - asset_name = os.path.basename(normalize_file_path(big_data_file_path)) - url = gen_file_get_url(token, asset_name) - - resp = requests.get(url) - page_json = json.loads(resp.content) - - - # b. poster - - poster_asset_id = seafile_api.get_file_id_by_path(repo_id, poster_asset_path) - poster_token = seafile_api.get_fileserver_access_token( - repo_id, poster_asset_id, 'view', '', use_onetime=False - ) - poster_name = os.path.basename(normalize_file_path(big_data_poster_path)) - url = gen_file_get_url(poster_token, poster_name) - resp_poster = requests.get(url) - - # 2. get the image infos in big-data-screen - page_bg_custom_image_list = page_json.get('page_bg_custom_image_list') - page_images = [] - page_elements = page_json.get('page_elements') or {} - element_map = page_elements.get('element_map') or {} - for key, value in element_map.items(): - if value.get('element_type') == 'image': - image_url = value.get('config',{}).get('imageUrl') - if "?" not in image_url: - page_images.append(image_url) - - - - content_json = { - 'page_content': page_json, - 'page_images': page_images - } - - # 3. write json file to tmp_file_path , write images to tmp_file_path/images - content_json_save_path = tmp_file_path.rstrip('/') - image_save_path = os.path.join(content_json_save_path, 'images') - with open("%s/content.json" % content_json_save_path, 'wb') as f: - f.write(json.dumps(content_json).encode('utf-8')) - with open("%s/content.png" % content_json_save_path, 'wb') as f: - f.write(resp_poster.content) - - for image_url in page_bg_custom_image_list: - target_path = normalize_file_path(os.path.join(base_dir, image_url.strip('/'))) - asset_id = seafile_api.get_file_id_by_path(repo_id, target_path) - if not asset_id: - continue - token = seafile_api.get_fileserver_access_token( - repo_id, asset_id, 'download', username, use_onetime=False - ) - image_name = os.path.basename(normalize_file_path(image_url)) - url = gen_file_get_url(token, image_name) - - resp = requests.get(url) - with open('%s/%s' % (image_save_path, image_name), 'wb') as f: - f.write(resp.content) - - for image_url in page_images: - target_path = normalize_file_path(os.path.join(base_dir, image_url.strip('/'))) - asset_id = seafile_api.get_file_id_by_path(repo_id, target_path) - if not asset_id: - continue - - token = seafile_api.get_fileserver_access_token( - repo_id, asset_id, 'download', username, use_onetime=False - ) - image_name = os.path.basename(normalize_file_path(image_url)) - url = gen_file_get_url(token, image_name) - - resp = requests.get(url) - with open('%s/%s' % (image_save_path, image_name), 'wb') as f: - f.write(resp.content) - -def post_big_data_screen_zip_file(username, repo_id, dtable_uuid, page_id, tmp_extracted_path): - - content_json_file_path = os.path.join(tmp_extracted_path, 'content.json') - content_poster_file_path = os.path.join(tmp_extracted_path, 'content.png') - new_content_poster_file_path = os.path.join(tmp_extracted_path, '%s.png' % page_id) - poster_file_name = os.path.basename(new_content_poster_file_path) - os.rename(content_poster_file_path, new_content_poster_file_path) - image_path = os.path.join(tmp_extracted_path, 'images/') - with open(content_json_file_path, 'r') as f: - content_json = f.read() - try: - content = json.loads(content_json) - except: - content = {} - - - base_dir = '/asset/' + dtable_uuid - big_data_file_path = 'files/plugins/big-data-screen/%(page_id)s/' % ({ - 'page_id': page_id, - }) - image_file_path = 'files/plugins/big-data-screen/bg_images/' - current_file_path = os.path.join(base_dir, big_data_file_path) - current_image_path = os.path.join(base_dir, image_file_path) - - # 1. handle page_content - page_content_dict = content.get('page_content') - page_content_dict['page_id'] = page_id # update page_id - tmp_page_json_path = os.path.join(tmp_extracted_path, '%s.json' % page_id) - with open(tmp_page_json_path, 'wb') as f: - f.write(json.dumps(page_content_dict).encode('utf-8')) - - path_id = seafile_api.get_dir_id_by_path(repo_id, current_file_path) - if not path_id: - seafile_api.mkdir_with_parents(repo_id, '/', current_file_path[1:], username) - file_name = os.path.basename(tmp_page_json_path) - dtable_file_id = seafile_api.get_file_id_by_path( - repo_id, current_file_path + file_name) - if dtable_file_id: - seafile_api.del_file(repo_id, current_file_path, json.dumps([file_name]), '') - seafile_api.post_file(repo_id, tmp_page_json_path, current_file_path, file_name, username) - seafile_api.post_file(repo_id, new_content_poster_file_path, current_file_path, poster_file_name, username) - - # 2. handle images - image_path_id = seafile_api.get_dir_id_by_path(repo_id, current_image_path) - if not image_path_id: - seafile_api.mkdir_with_parents(repo_id, '/', current_image_path[1:], username) - for dirpath, _, filenames in os.walk(image_path): - for image_name in filenames: - tmp_image_path = os.path.join(dirpath, image_name) - dtable_file_id = seafile_api.get_file_id_by_path( - repo_id, current_image_path + image_name - ) - if not dtable_file_id: - seafile_api.post_file(repo_id, tmp_image_path, current_image_path, image_name, username) - - def zip_big_data_screen_app(username, repo_id, dtable_uuid, app_uuid, app_id, tmp_file_path, db_session): from dtable_events.utils import uuid_str_to_36_chars, normalize_file_path, gen_file_get_url