Skip to content

remove legacy big data screen plugin IO tasks #916

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
freeplant merged 1 commit into from
Oct 16, 2026
Merged

remove legacy big data screen plugin IO tasks #916

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 1 addition & 35 deletions dtable_events/dtable_io/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
prepare_dtable_json_from_memory, update_page_design_static_image, \
copy_src_auto_rules_to_json, create_auto_rules_from_src_dtable, sync_app_users_to_table, \
copy_src_workflows_to_json, create_workflows_from_src_dtable, copy_src_external_app_to_json,\
create_external_apps_from_src_dtable, zip_big_data_screen, post_big_data_screen_zip_file, \
create_external_apps_from_src_dtable, \
export_page_design_dir_to_path, update_page_design_content_to_path, upload_page_design, \
download_page_design_file, zip_big_data_screen_app, prepare_asset_files_download, copy_src_archive_backup
from dtable_events.db import init_db_session_class
Expand Down Expand Up @@ -515,40 +515,6 @@ def get_dtable_export_asset_files(username, repo_id, dtable_uuid, files, task_id
finally:
db_session.close()

def get_dtable_export_big_data_screen(username, repo_id, dtable_uuid, page_id, task_id):
"""
parse json file in big data screen, and zip it for download
"""
tmp_file_path = os.path.join('/tmp/dtable-io', dtable_uuid, 'big-data-screen', str(task_id))
tmp_zip_path = os.path.join('/tmp/dtable-io', dtable_uuid, 'big-data-screen', str(task_id) + '.zip')
clear_tmp_files_and_dirs(tmp_file_path, tmp_zip_path)
os.makedirs(tmp_file_path.rstrip('/') + '/images', exist_ok=True)

try:
zip_big_data_screen(username, repo_id, dtable_uuid, page_id, tmp_file_path)
shutil.make_archive(tmp_zip_path.split('.')[0], 'zip', root_dir=tmp_file_path)
except Exception as e:
dtable_io_logger.error('export big data screen from dtable failed. ERROR: {}'.format(e))
else:
dtable_io_logger.info('export big data screen from dtable: %s success!', dtable_uuid)

def import_big_data_screen(username, repo_id, dtable_uuid, page_id):
"""
parse the zip in tmp folders and upload it
"""
tmp_extracted_path = os.path.join('/tmp/dtable-io', dtable_uuid, 'big_data_screen_zip_extracted/')
try:
post_big_data_screen_zip_file(username, repo_id, dtable_uuid, page_id, tmp_extracted_path)
except Exception as e:
dtable_io_logger.error('import big data screen from dtable failed. ERROR: {}'.format(e))
else:
dtable_io_logger.info('import big data screen to dtable: %s success!', dtable_uuid)
try:
shutil.rmtree(tmp_extracted_path)
except Exception as e:
dtable_io_logger.error('rm extracted tmp file failed. ERROR: {}'.format(e))


def get_dtable_export_big_data_screen_app(username, repo_id, dtable_uuid, app_uuid, app_id, task_id):
"""
parse json file in big data screen, and zip it for download
Expand Down
53 changes: 0 additions & 53 deletions dtable_events/dtable_io/request_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,59 +118,6 @@ def add_import_task():

return make_response(({'task_id': task_id}, 200))

@app.route('/add-big-data-screen-export-task', methods=['GET'])
def add_big_data_screen_export_task():
is_valid, error = check_auth_token(request)
if not is_valid:
return make_response((error, 403))

if task_manager.tasks_queue.full():
dtable_io_logger.warning('dtable io server busy, queue size: %d, current tasks: %s, threads is_alive: %s'
% (task_manager.tasks_queue.qsize(), task_manager.current_task_info,
task_manager.threads_is_alive()))
return make_response(('dtable io server busy.', 400))

username = request.args.get('username')
repo_id = request.args.get('repo_id')
dtable_uuid = request.args.get('dtable_uuid')
page_id = request.args.get('page_id')

try:
task_id = task_manager.add_export_dtable_big_data_screen_task(
username, repo_id, dtable_uuid, page_id)
except Exception as e:
dtable_io_logger.error(e)
return make_response((e, 500))

return make_response(({'task_id': task_id}, 200))

@app.route('/add-big-data-screen-import-task', methods=['GET'])
def add_big_data_screen_import_task():
is_valid, error = check_auth_token(request)
if not is_valid:
return make_response((error, 403))

if task_manager.tasks_queue.full():
dtable_io_logger.warning('dtable io server busy, queue size: %d, current tasks: %s, threads is_alive: %s'
% (task_manager.tasks_queue.qsize(), task_manager.current_task_info,
task_manager.threads_is_alive()))
return make_response(('dtable io server busy.', 400))

username = request.args.get('username')
repo_id = request.args.get('repo_id')
dtable_uuid = request.args.get('dtable_uuid')
page_id = request.args.get('page_id')

try:
task_id = task_manager.add_import_dtable_big_data_screen_task(
username, repo_id, dtable_uuid, page_id)
except Exception as e:
dtable_io_logger.error(e)
return make_response((e, 500))

return make_response(({'task_id': task_id}, 200))


@app.route('/add-big-data-screen-app-export-task', methods=['GET'])
def add_big_data_screen_app_export_task():
is_valid, error = check_auth_token(request)
Expand Down
22 changes: 0 additions & 22 deletions dtable_events/dtable_io/task_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,28 +117,6 @@ def add_export_dtable_asset_files_task(self, username, repo_id, dtable_uuid, fil
publish_metric(self.tasks_queue.qsize(), 'io_task_queue_size', metric_help=TASK_MANAGER_METRIC_HELP)
return task_id

@log_function_call
def add_export_dtable_big_data_screen_task(self, username, repo_id, dtable_uuid, page_id):
from dtable_events.dtable_io import get_dtable_export_big_data_screen
task_id = str(uuid.uuid4())
task = (get_dtable_export_big_data_screen,
(username, repo_id, dtable_uuid, page_id, task_id))
self.tasks_queue.put(task_id)
self.tasks_map[task_id] = task
publish_metric(self.tasks_queue.qsize(), 'io_task_queue_size', metric_help=TASK_MANAGER_METRIC_HELP)
return task_id

@log_function_call
def add_import_dtable_big_data_screen_task(self, username, repo_id, dtable_uuid, page_id):
from dtable_events.dtable_io import import_big_data_screen
task_id = str(uuid.uuid4())
task = (import_big_data_screen,
(username, repo_id, dtable_uuid, page_id))
self.tasks_queue.put(task_id)
self.tasks_map[task_id] = task
publish_metric(self.tasks_queue.qsize(), 'io_task_queue_size', metric_help=TASK_MANAGER_METRIC_HELP)
return task_id

@log_function_call
def add_export_dtable_big_data_screen_app_task(self, username, repo_id, dtable_uuid, app_uuid, app_id):
from dtable_events.dtable_io import get_dtable_export_big_data_screen_app
Expand Down
150 changes: 0 additions & 150 deletions dtable_events/dtable_io/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1622,156 +1622,6 @@ def height_transfer(base_row_height='default'):

return round((32 * row_height_mul * 14.4 ) / 24, 2)

def zip_big_data_screen(username, repo_id, dtable_uuid, page_id, tmp_file_path):
from dtable_events.utils import uuid_str_to_36_chars, normalize_file_path, gen_file_get_url
base_dir = '/asset/' + dtable_uuid
big_data_file_path = 'files/plugins/big-data-screen/%(page_id)s/%(page_id)s.json' % ({
'page_id': page_id,
})
big_data_poster_path = 'files/plugins/big-data-screen/%(page_id)s/%(page_id)s.png' % ({
'page_id': page_id
})

asset_path = "%s/%s" % (base_dir, big_data_file_path)

poster_asset_path = "%s/%s" % (base_dir, big_data_poster_path)

# 1. get the json file and poster of big-data-screen page
# a. json file
asset_id = seafile_api.get_file_id_by_path(repo_id, asset_path)
token = seafile_api.get_fileserver_access_token(
repo_id, asset_id, 'view', '', use_onetime=False
)
asset_name = os.path.basename(normalize_file_path(big_data_file_path))
url = gen_file_get_url(token, asset_name)

resp = requests.get(url)
page_json = json.loads(resp.content)


# b. poster

poster_asset_id = seafile_api.get_file_id_by_path(repo_id, poster_asset_path)
poster_token = seafile_api.get_fileserver_access_token(
repo_id, poster_asset_id, 'view', '', use_onetime=False
)
poster_name = os.path.basename(normalize_file_path(big_data_poster_path))
url = gen_file_get_url(poster_token, poster_name)
resp_poster = requests.get(url)

# 2. get the image infos in big-data-screen
page_bg_custom_image_list = page_json.get('page_bg_custom_image_list')
page_images = []
page_elements = page_json.get('page_elements') or {}
element_map = page_elements.get('element_map') or {}
for key, value in element_map.items():
if value.get('element_type') == 'image':
image_url = value.get('config',{}).get('imageUrl')
if "?" not in image_url:
page_images.append(image_url)



content_json = {
'page_content': page_json,
'page_images': page_images
}

# 3. write json file to tmp_file_path , write images to tmp_file_path/images
content_json_save_path = tmp_file_path.rstrip('/')
image_save_path = os.path.join(content_json_save_path, 'images')
with open("%s/content.json" % content_json_save_path, 'wb') as f:
f.write(json.dumps(content_json).encode('utf-8'))
with open("%s/content.png" % content_json_save_path, 'wb') as f:
f.write(resp_poster.content)

for image_url in page_bg_custom_image_list:
target_path = normalize_file_path(os.path.join(base_dir, image_url.strip('/')))
asset_id = seafile_api.get_file_id_by_path(repo_id, target_path)
if not asset_id:
continue
token = seafile_api.get_fileserver_access_token(
repo_id, asset_id, 'download', username, use_onetime=False
)
image_name = os.path.basename(normalize_file_path(image_url))
url = gen_file_get_url(token, image_name)

resp = requests.get(url)
with open('%s/%s' % (image_save_path, image_name), 'wb') as f:
f.write(resp.content)

for image_url in page_images:
target_path = normalize_file_path(os.path.join(base_dir, image_url.strip('/')))
asset_id = seafile_api.get_file_id_by_path(repo_id, target_path)
if not asset_id:
continue

token = seafile_api.get_fileserver_access_token(
repo_id, asset_id, 'download', username, use_onetime=False
)
image_name = os.path.basename(normalize_file_path(image_url))
url = gen_file_get_url(token, image_name)

resp = requests.get(url)
with open('%s/%s' % (image_save_path, image_name), 'wb') as f:
f.write(resp.content)

def post_big_data_screen_zip_file(username, repo_id, dtable_uuid, page_id, tmp_extracted_path):

content_json_file_path = os.path.join(tmp_extracted_path, 'content.json')
content_poster_file_path = os.path.join(tmp_extracted_path, 'content.png')
new_content_poster_file_path = os.path.join(tmp_extracted_path, '%s.png' % page_id)
poster_file_name = os.path.basename(new_content_poster_file_path)
os.rename(content_poster_file_path, new_content_poster_file_path)
image_path = os.path.join(tmp_extracted_path, 'images/')
with open(content_json_file_path, 'r') as f:
content_json = f.read()
try:
content = json.loads(content_json)
except:
content = {}


base_dir = '/asset/' + dtable_uuid
big_data_file_path = 'files/plugins/big-data-screen/%(page_id)s/' % ({
'page_id': page_id,
})
image_file_path = 'files/plugins/big-data-screen/bg_images/'
current_file_path = os.path.join(base_dir, big_data_file_path)
current_image_path = os.path.join(base_dir, image_file_path)

# 1. handle page_content
page_content_dict = content.get('page_content')
page_content_dict['page_id'] = page_id # update page_id
tmp_page_json_path = os.path.join(tmp_extracted_path, '%s.json' % page_id)
with open(tmp_page_json_path, 'wb') as f:
f.write(json.dumps(page_content_dict).encode('utf-8'))

path_id = seafile_api.get_dir_id_by_path(repo_id, current_file_path)
if not path_id:
seafile_api.mkdir_with_parents(repo_id, '/', current_file_path[1:], username)
file_name = os.path.basename(tmp_page_json_path)
dtable_file_id = seafile_api.get_file_id_by_path(
repo_id, current_file_path + file_name)
if dtable_file_id:
seafile_api.del_file(repo_id, current_file_path, json.dumps([file_name]), '')
seafile_api.post_file(repo_id, tmp_page_json_path, current_file_path, file_name, username)
seafile_api.post_file(repo_id, new_content_poster_file_path, current_file_path, poster_file_name, username)

# 2. handle images
image_path_id = seafile_api.get_dir_id_by_path(repo_id, current_image_path)
if not image_path_id:
seafile_api.mkdir_with_parents(repo_id, '/', current_image_path[1:], username)
for dirpath, _, filenames in os.walk(image_path):
for image_name in filenames:
tmp_image_path = os.path.join(dirpath, image_name)
dtable_file_id = seafile_api.get_file_id_by_path(
repo_id, current_image_path + image_name
)
if not dtable_file_id:
seafile_api.post_file(repo_id, tmp_image_path, current_image_path, image_name, username)


def zip_big_data_screen_app(username, repo_id, dtable_uuid, app_uuid, app_id, tmp_file_path, db_session):
from dtable_events.utils import uuid_str_to_36_chars, normalize_file_path, gen_file_get_url

Expand Down
Loading