From cb7dccb9ebdefe6dfa6a7ea346e6a6fef3d9d308 Mon Sep 17 00:00:00 2001 From: Ivan Marcellino Date: Fri, 21 Jun 2024 12:48:21 +0700 Subject: [PATCH 1/7] Scripts to convert COCO <-> Datasaur Schema (#30) * handles COCO <-> Datasaur Schema conversion * custom attributes handling * also: multiple project from multiple folders --- .gitignore | 1 + bounding-boxes/readme.md | 155 +++++++++ bounding-boxes/samples/COCO.json | 70 +++++ bounding-boxes/samples/bbox-NDU4OWNmZTA.zip | Bin 0 -> 4099 bytes bounding-boxes/samples/bbox-export.zip | Bin 0 -> 9865 bytes bounding-boxes/samples/custom-label-set.json | 42 +++ bounding-boxes/samples/license-and-info.json | 17 + bounding-boxes/src/__init__.py | 2 + .../src/coco_to_datasaur_schemas.py | 294 ++++++++++++++++++ bounding-boxes/src/common/defaults.py | 2 + bounding-boxes/src/common/logger.py | 23 ++ bounding-boxes/src/common/random_color.py | 7 + bounding-boxes/src/common/scrub.py | 24 ++ .../src/datasaur_schemas_to_coco.py | 276 ++++++++++++++++ bounding-boxes/src/formats/bbox_labelset.py | 81 +++++ bounding-boxes/src/formats/coco.py | 98 ++++++ bounding-boxes/src/formats/datasaur_schema.py | 98 ++++++ create-project-async/api_client.py | 44 ++- create-project-async/config.csv | 2 + create-project-async/readme.md | 22 +- create-project-async/src/helper.py | 18 +- create-project-async/src/logger.py | 23 ++ create-project-async/src/project.py | 64 ++-- 23 files changed, 1333 insertions(+), 30 deletions(-) create mode 100644 bounding-boxes/readme.md create mode 100644 bounding-boxes/samples/COCO.json create mode 100644 bounding-boxes/samples/bbox-NDU4OWNmZTA.zip create mode 100644 bounding-boxes/samples/bbox-export.zip create mode 100644 bounding-boxes/samples/custom-label-set.json create mode 100644 bounding-boxes/samples/license-and-info.json create mode 100644 bounding-boxes/src/__init__.py create mode 100644 bounding-boxes/src/coco_to_datasaur_schemas.py create mode 100644 bounding-boxes/src/common/defaults.py create mode 100644 bounding-boxes/src/common/logger.py create mode 100644 bounding-boxes/src/common/random_color.py create mode 100644 bounding-boxes/src/common/scrub.py create mode 100644 bounding-boxes/src/datasaur_schemas_to_coco.py create mode 100644 bounding-boxes/src/formats/bbox_labelset.py create mode 100644 bounding-boxes/src/formats/coco.py create mode 100644 bounding-boxes/src/formats/datasaur_schema.py create mode 100644 create-project-async/config.csv create mode 100644 create-project-async/src/logger.py diff --git a/.gitignore b/.gitignore index 51a41e7..589bdf7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ readme.txt output/ __pycache__ .env +bounding-boxes/outdir/ \ No newline at end of file diff --git a/bounding-boxes/readme.md b/bounding-boxes/readme.md new file mode 100644 index 0000000..39eae15 --- /dev/null +++ b/bounding-boxes/readme.md @@ -0,0 +1,155 @@ +# Bounding Boxes Formats: COCO + +This folder contains Python scripts for converting annotation results from the COCO format to Datasaur Schema format and vice versa. + +- [Formats](#formats) + - [COCO](#coco) + - [Datasaur Schema](#datasaur-schema) +- [Prerequisite](#prerequisite) +- [Usage](#usage) + - [`coco_to_datasaur_schemas`](#coco_to_datasaur_schemas) + - [`datasaur_schemas_to_coco`](#datasaur_schemas_to_coco) + + +## Formats + +### COCO + +[COCO (Common Objects in Context)](https://cocodataset.org/#home) has formats for several annotation tasks. +The script here will only support cases where the `segmentation` mask are in a quadrilateral shape. +In other words, it will only work for cases where there are 8 values per list in the `segmentation` key. + +```json +{ + "annotations": [ + { + "segmentation": [["x1", "y1", "x2", "y2", "x3", "y3", "x4", "y4", "x5", "y5", "x6", "y6", "x7", "y7", "x8", "y8"]] + } + ] +} +``` + +### Datasaur Schema + +[Datasaur Schema](https://docs.datasaur.ai/compatibility-and-updates/supported-formats#datasaur-schema-format) is a customized format Datasaur uses for both creating and exporting a Datasaur project. + + +## Prerequisite + +The script was developed and tested using Python 3.10.13 + +## Usage + +### `coco_to_datasaur_schemas` + +This function transforms a dictionary representation of a COCO file into a list of `DatasaurSchema` dicts. + +Parameters: +- `coco_json`: A dictionary representation of a COCO file. +- `custom_labelset`: Optional. A dictionary representation of a Datasaur bounding box label set to provide additional information. Useful to set default value, custom color, or setting a specific attribute as `DROPDOWN` type. See [here](./samples/custom-label-set.json) for an example JSON file. + > [!IMPORTANT] + > You can obtain a custom labelset JSON by copy-pasting from Datasaur's BBox Label Set editor. Please keep in mind that there are some attributes that won't be parsed, such as label's `id` and question's `internalId`, as they will be generated upon creation. + > Other than that, the script will also ignore the following TEXT config: `minLength`, `maxLength` and `pattern` + +Running `python src/coco_to_datasaur_schemas.py` should run the function against a sample `COCO.json` file. The function expect a `dict` representation of a COCO file and will return an array of `DatasaurSchema` also in a `dict` format. + +Note: The function will ignore the following fields as they are not used in Datasaur Schema: +- `licenses` +- `info` +- `annotation.area`, `annotation.bbox` + +Example usage: +- as a function + ```python + coco_json = json.load(file) + datasaur_schemas = coco_to_datasaur_schemas(coco_json) + ``` +- as a script + ``` + $ usage: coco_to_datasaur_schemas [-h] [--custom-labelset CUSTOM_LABELSET] [--outdir OUTDIR] [--log-level LOG_LEVEL] [--ignored-attributes IGNORED_ATTRIBUTES] coco_filepath + + positional arguments: + coco_filepath Path to COCO JSON file + + options: + -h, --help show this help message and exit + --custom-labelset CUSTOM_LABELSET + Path to custom labelset JSON file (useful for specifying DROPDOWN attributes) + --outdir OUTDIR Output directory for Datasaur schemas + --log-level LOG_LEVEL + --ignored-attributes IGNORED_ATTRIBUTES + Comma separated strings of attribute keys to ignore. Default: occluded + + $ python src/coco_to_datasaur_schemas.py samples/COCO.json + ``` + +### `datasaur_schemas_to_coco` + +This function transforms a list of Datasaur Schema object, represented as a list of dictionaries, into a single COCO object. + +Parameters: +- `schema_objects` (list[Any]): A list of dictionaries representing the result of `json.load`-ing the Datasaur Schema JSON. +- `licenses` (list[COCOLicense] | None, optional): A list of COCOLicense objects. Defaults to None. +- `info` (COCOInfo | None, optional): A COCOInfo object. Defaults to None. + +Running `python src/datasaur_schemas_to_coco.py` will read the sample zipfile `samples/bbox-export.zip` and transform the JSON file under the `REVIEW` directory into a single `COCO` object, which will be written to `outdir/out-coco.json`. + +As `licenses` and `info` parameters are optional, when not provided it will use the following values: + +```json +{ + "licenses": [ + { + "name": "dummy-datasaur-license", + "id": 0, + "URL": "" + } + ], + "info": { + "description": "Exported from Datasaur", + "url": "https://datasaur.ai", + "version": "v0.1", + "year": 2024, + "contributor": "Datasaur", + "date_created": "2024-04-23" + } +} +``` + + +Example usage: +- as a function + ```python + schemas = [...] + coco_obj = datasaur_schemas_to_coco(schemas) + + # or provide your own `licenses` and `info` + coco_obj = datasaur_schemas_to_coco( + schemas, + licenses=[{"name": "dummy-license", "id": 0, "url": ""}], + info={ + "contributor": "contributor", + "date_created": "2024-01-01", + "description": "dataset-description", + "url": "http://example.com", + "version": "v0.1", + "year": 2024, + }, + ) + ``` +- as a script + ``` + $ python src/datasaur_schemas_to_coco.py -h + usage: datasaur_schemas_to_coco [-h] [--outfile OUTFILE] [--license-and-info-json LICENSE_AND_INFO_JSON] zip_filepath + + positional arguments: + zip_filepath Path to Datasaur export ZIP file + + options: + -h, --help show this help message and exit + --outfile OUTFILE Output directory for COCO JSON file + --license-and-info-json LICENSE_AND_INFO_JSON + Path to JSON file containing licenses and info data + + $ python src/datasaur_schemas_to_coco.py samples/bbox-export.zip + ``` diff --git a/bounding-boxes/samples/COCO.json b/bounding-boxes/samples/COCO.json new file mode 100644 index 0000000..b0f0e18 --- /dev/null +++ b/bounding-boxes/samples/COCO.json @@ -0,0 +1,70 @@ +{ + "licenses": [ + { + "name": "DATASAUR", + "id": 0, + "URL": "datasaur.ai" + } + ], + "info": { + "contributor": "datasaur.ai", + "date_created": "2024-04-23", + "description": "Description of the dataset", + "url": "", + "version": "v0.1", + "year": 2024 + }, + "categories": [ + { + "id": 1, + "name": "sample_category", + "supercategory": "" + }, + { + "id": 2, + "name": "merchant_name", + "supercategory": "" + } + ], + "images": [ + { + "id": 1, + "width": 239.0, + "height": 991.0, + "file_name": "sample.png", + "license": 0, + "flickr_url": "", + "coco_url": "", + "date_captured": 0 + } + ], + "annotations": [ + { + "id": 1, + "image_id": 1, + "category_id": 1, + "segmentation": [[73, 153, 113, 153, 113, 169, 74, 169]], + "bbox": [73, 153, 40, 16], + "area": 7938, + "iscrowd": 0, + "attributes": { + "text": "sample text" + } + }, + { + "id": 2, + "image_id": 1, + "category_id": 2, + "segmentation": [[78, 158, 118, 158, 118, 174, 79, 174]], + "bbox": [78, 158, 40, 16], + "area": 7938, + "iscrowd": 0, + "attributes": { + "text": "Hokkaido-Ya", + "bio_tag": "B", + "order_idx": "1", + "occluded": false + } + } + ] +} diff --git a/bounding-boxes/samples/bbox-NDU4OWNmZTA.zip b/bounding-boxes/samples/bbox-NDU4OWNmZTA.zip new file mode 100644 index 0000000000000000000000000000000000000000..c560784276f73d0393754543d6988e5fbb458c79 GIT binary patch literal 4099 zcmb_fXIPVI77bOJh*G2)RGJhK5u_V>2_aM?HAXrDfrKtC3JeGYL^@KWNE47?fCUvt zB_J9~j7l+Jp$|m_1eCHwmyy|ld1lULlhw1O> zI?Mi;PI0GnI#6vZh^~pHjC%mW8}5ZbIXZiKx_cwxP6$5)8sU$EBiz4ZjvUbYYfL5p z6JTH)W(xxVp!?I9XoQ!Kr}J?SAJ^j^Xr%YIw%0dHpguda-D>%($pDDs{*qMTw1Qau zhis+N0}SN`TpY!;c&(Q~fr9DPfyDk89f^%Wp@GB9RZQxOqhYG^qr^+~`F%(G&H0H0 z>GNS?1+L^&|V1I=aqlV z!0e`aPnLBOAED?@*NHvlg7KmBVaZoU%2LUr@?PdzUi&~K2b?sQbXR|zcy;~KVo5rI z=Y&50*EedpZWva@&$g?q9V@v5$FVAJZLUsME7#mO<&EmkYoe6b7rUs3sqv*+KJ>KP!AFY+k zT)W(hm3g)-kUY`aRWeI(C!mW^mn8|7G*n_Xsj4<{7x%>|6%)}NK{FjdcrrhON25_PP%GgE()okw;WYabNS!`abKhRU2DGfSoF)17Cw@~ospN0$-g zxEZeEY=^NK&ioUaSz&J9(z8SsUrUmiA4hbKGe@0 zRpq}KPT~Xr#Q!4eZXC|v1lWPY(=dB&NsI{Yo_yoVUs=#1{*VgCD8#AY*UI;mwC|Vj4h{_BoqYw zrG5k(%gOH5g^OV)tar}~BF8yAgY%;Ai?Z#t=;|4?xQ9K8S`c;M`T$TEfPxb z3t69=Ul?8sb+OAJFv%L~q%g1sOR-;?D)ivS5l)#u>$ZK4eWf&K$Yn-=S2`6dfw3nj z1g`E=*pH8``TDgwX^@r5#N`Y3js%A$>0bJk7A5;N-Fjg|#LvG7cPe0^n|tpQh#akA zh?EAr(r>hS;avK#+;6;aAz&#j>m;!9zEDz|h6tyuph~(_oq$r&VFKoij-rE2`;|@- z%cs!vLGns&Mbv%q){`Rv{{5C5`%Ns&Pb<WV|8)?+FO)o7 z4njo*#m?K=cb;(w2>z(dI#EHO#P}!9LYZwxoDkMrcp(W6=OLPoBbuVYRwoW)N!W7= zMir)4Y%+`MA>FH(F~hX;;g1x zjRBf>C_1V#Ub1D+SUo}}d{ArGJF9wHsu%fhE-^)L9S-w>YLa7%}4t+~~F=?N?YPFtP%lK651lxp&I1Q_!mp0M@_Nnz`XSo!T5 zY}zKJBD3^`*Mx=b;KQu9E6dGML!?NNk_1No(kA}O$T3|R&(1u~wRs9;mFEx#|o&5|c2vOt2qeFY7!rrG1`|00jI|R9{4Qih}_&DIc z;9{uUY7UcXck=+7;+2SR3I%!N7w~%1qEHp%heEY(a|=|@b8<&5fkOX=J0TtT33sCT zQ+T_n+OE#T9@!7#2%W6~y@qLcemF2ydO6mzn5!^K(oD)y=m8!*vC^qLKp(X_bm^i1 zn>+Wo#ZV}^xvoyWKG12e%81c{{N7>+FHvGHP^4#)?=l-uSw67t>FL1r6Y(vwdbf;7 zyljm&<$I4AUuXHm>tvzMORs1bjA+BhIJ7y>l(Lsdk1K^W;RL$^U_2My%=yJ>o4O36 zoLU;4j?G0R&jqb7qt%*P@{*l%n3`y5LL{~Ez!yNaTt!SK3UO{ZyTjYqAE4z^uJJ+a z7|}li^(MYFu3*4g!PmpX7QPR!qH%GFH89ZI_~qnBBg> zBx7{>L5K8T&F;aB3F#}|Q^WPw&Sxo$YrLddkfkSfnb~MDg)s?*F=LO;6>@!~reGUS z%&*;if^DN~0~s;$7eCK6u8{UJ6XmTAA@Me7#wL`I0Q*MNC%6R{Fr1G1HV13cI?DWR zL1<{X=yqQZsH~yJv?lfZiDltO`kmGVYEo)UYeF~AKc)XaEf3#o*fJN}9T;_e*otDz zQiZR)nn)at_`Nu|Z3y+$2C)c)xxA`~F?|}EWJ=~A3 zsC(LWO{TiIttiHn`~}KxSGkL)+uqtva=!&StvScEQ)~ab`2Vr#?c}1$|5g-Z5X?V= bzuR@}@-i?5GcZyisl(pAIXgmZ)L;Jw^zk@V literal 0 HcmV?d00001 diff --git a/bounding-boxes/samples/bbox-export.zip b/bounding-boxes/samples/bbox-export.zip new file mode 100644 index 0000000000000000000000000000000000000000..ae234b962e576aefdd351e7c9d6183867ca05848 GIT binary patch literal 9865 zcmb_iWmp{9wgnmp4#6Elf=dYQ?k>R{0>NE^)3^j0C%6T7cMZYagS$&01P$<-$(@^- z%$>RSeKV)O?yCM#dso$2Yu8%4j)F7*8WZASIN2tt|FZM1AAIoE!NAJa(wN@N%HXxJ z1HHV6tec#OD6@vS#dC!}J&yk`j}ws>R*@Bzdr5ERY+$WtWngb)Y-wp`ZKLviFaZv07dt`rFKcBzk_dU%jM zJKYONKN2Q_00#ktd`yc6{XBB>G%6^^O@<6kpRMuMfeSz8IcT{ZqJxbPifWcZx*^eW zHWa6voiy5@W;M$l`HCY;ZYK$|Vc~t|kn9$Qxssgcj6ElONj-NNU$2N72v4tIy_KAd zhV&A_ujAkgiQk8Pj)afGU1joer)*CbtFsK${E9MkHCW|zq7go!Ri`c>$#$#*KYRnnE5adYlbaL%~4Ga;nk$e*JqvDT-7?v`Dp4R zdjNeS#hK_y*B8#Rpe0|3cRLlD`*D>xsgv_JrQEQv88ad6_S7!yGy$B?L9`#@kDW;R z^hXb&0O~6J*V{S_b&_3Q8WX=RZhm5+zP!gf$#6~|A`H>c16#dK-UjR) zW#Wi$Fxu;QoL$Kd+tz<@6`lh1Nj{aE$vv*l>BE460EDs*EL}WJgyUuBHWp}w96@^{ zAviB65cQ?ihiY2tCPK*^p+Bt<3Tg*6XuTpiOhy*cBz8bQM zaYMtkm~A-9!Kto!x=9|Zq18cS*!z~l^sNFi@>6M}uC#2dIhxQwrTmGk^RL8G4B>P4 z%Nb&`Rv8-3h|}Zx^ldQvuK1SR+Y%$36i3F2?$ZH_>qwKeQyf6=l~60$B*OZ=4oWNb zGwFBQb9^c3yb~MCg8M~>5RF$x3l;&)`4fQ|i<*_fS}tZ4+|E@&gs=;>XT;RVd_EO8n)eU+WcB} zzE=jPyzLV+aAnv;_y;Qk(~rt9r1DE;@H|$7t163r8q#=L&K$?=IJZ1H0!m8sb) zX4;rShKPywdtC*b0l7+1C`^gv9p;UcL zRNC^TcWLsu8#lM|qk`r1PyxPHRhY=F#ujud8Nzhg=3!ahTNnf6KoIPS;fU=!0=&Vm zA&*5Wguk_$==yuMazz=S9x-bdIr29IBWCdHvozkFZ=2S?lM@K)`+7Mr<>)m7B~7GM z+ll!lQQ}q4!93SW{K?Cl#TPGTD|tFuHihm=-N01=6<3Yu3z_t%^=4{=S|gz*t|b+f zS57WxJ<6U*A-ER;L$LRND>r4R8ku>s$TA)Vc4S-iJ+!_ObdeUlJ%!D}w1{+FZA@}n zSI5U2S8igs0 z5_Un{AvA=clQcW8iCMXN4durwz zhzX|yuiy4I0kskoqqdez_&7G~Ilk681(N8A&lS{uCOZ(S2$DXkVR~^4vuO&>Wz?W1 z`bPR=P5Wsz%si6cods_AwN?h8NJ`_rxoYJDw(u_!QaH`pS;Ilb(t z6k8NNQ7UdlIVEzG4Xbl6H^_{J^6+> z;UN5@CyJPX#~bQ21R&iMs@Gb;&2*2~RW8MqMy(a6M$RT(u9kP9^s=b}1{uh(8Nw9? zpY{jD?Z<|PiK$LJh=_sF?JxTefsM~DVb*3Y{W-gLhhs}1G@su}Xn~_kj_`$f`;rJ4qb3;>xJ9lyS@AC=MwP7tB)9nz$ zknpY!Kazjbuo|EmREfMK2;cGEAk z-e8oDq9Ny$1GKpd647zn;UJf{FZJC=I$U~1dj${``bt7VTQZVnlEa)LG(^keCS1pm z3+QvU+`fL1+ThLS2>?J3p}}FY3~(mVWoyx`=nIgo(+H2*ARdfH(7hd``V5$;qGQQ! zuN`C*8d$tsMFW(Ls8KR47UdDvaV@e_L__te;erzM?(jjXRn)kQI&yrA!`P@L(})cxMz-zZA!-)T_MdeX_i}n_u+R^>yhO4FSvcXi z@)BKLYkCif`nr>6ZDQax5CG?$yVcB&HsYQj0H z&n_qhiP%JsxE&(ETTR9Q>itVBe1+Bxw<9)u>8Q$*z;|P}$e|@^qAybjB43<{<-h6U zz<_&})7-Aw=SoS4A~q3syR0J{H;8dXH_T-*QBxv&oZvFCx&2YYor#~MnXg$Vio#S1 zI9dZSeDmgt?xx=T$z490VAG{_xsy$&1m;M-syc}qBHXAZ?t3NVMpi)TqIEkF(yZmi zo!E?~fh(7_P>!%Hfj^bb=17v)`~ zN<-B@A|=b9SXrprFQZtgbn0?aa_V5m8Zp?fI<835N7pB*QmI5EH%Q+rIW;sWD5+%~+Z7Gvg%XNK^G++h;m9 zd0fO91qsYA61|TaZ4GEUbht`gH`3Mj6HP=wc`~anFh~QtintN*rYk=>-3;l8j5xdZ zQbhOJz&bmlPZg^W9J$suUeaz7eT9E2WJT~E>*>+EebLfm&M3S`*sQ7k&?Y`(o!a%7 zFYF>I_;wFi17&X3Ui9R!T+naY!LO>}qU5FDc!7t?q>2?@p@3t1Y^ zdW|b3(6@SO8_J0W9O*#syvl=Rk}g~27pTg|RP?z9+h){N3sz{j5^Hk|1mK>w^bOkj zY0Nf}c(*+t4?SnF0DyxT zV7uXy+8HK)Jk_AY$F!fDq?rhvo7y2%)v~UxD$tTyy0fUaEZJHKv{|u+)HT*JIrru| zI&v(ct<@!46C+{yy5Nz6$+4#WIeM8+CRxK(k0WSnUw^94DAVbsL)CsJxe;>*YKlCq zc(ffJl&k-Uk19Nilw+(%Th&D`3}D}N&f{i#OF=19-_k;>0qRqH5;0@FOdS@AAp`;F z#~N5p7bn|nw5hd+fPv)^-kC?A!xri}YI{D5GA89NMuQB)nYM>6R8Nf5Gc$CiQpHBh zCk*EobhDdzreOF)S@{c4^zqh|R*S$@CD>AgnY#mBbo;@U!o+YMnaNNKpX}sD(@s;A zPfW?~ntn&%*lveY@FxCi8>AJ6{FAB$?4q0sO~J6K3)bn{9;?N5f46NI&#eYKDAO4w zX3|+@^3pAti za+xA-w6UHUclWK5dt7$>?bY6}w}MUj6|YOTXievuwW##62R%^7gw=8Ip z*wnyJk)|drLPgy=fx|3|x_WS*g1z->p5sOj(nP5yds@gj%@lCuY4dcTRHe|uLlZIr z$9$n&eOZ(*aB;4C@zCCZ%I=fGDVOSQNc4T*Ye7SCF6Khfx8_(3uIfW6+ck)O;+0lk z$=+8UFk!3mz3lQ7nYPTqZ^OFYT<5b+L%4&f>;4(p9(tD@vZ@yj#_SZn66ju4xF@ z97WRaj`CSbK`u^MW>Z90cP~?OqgRQ8b=KRS=-D8UMjT$Vo(5DIDg%OY;49|L10QWXoI=-CGQYP%P-zS*$WKFU^ILPFU9M6t@rH)6Y-4Wq$35pvoU0{S06&@`mJ$Ul&M@LT)wM%wVi#UtaVm2UGZtxU(NVB}80hB` z{6Z){;R?iP&}l7flES2auK<=9bSGpp({Lq8#0Z(hjk~6!$q%+c1#`&(p=2LkG`cHu z-kUoha0Krg4pJO5C1Dy4SYgOTjSDw_z>)o+qu*Dq47;~xC+H&z_fb4OO~>Q#L>GF~b2qO`lCOBo z7gLb74}C8z=oibU#zPih;qkk^ew%~LTP7WPitLfU&~YzhJ>VI%pj(nM!yih(Bb*}m z^df^b!xPi%!10EJXYpg;wip%@8w)#n#{_2L<02Wu__jqq(!vDMNli2ktd-fLu~@fu zK${Nd?N;E$GIu!-q1P3`?>X!DZCsWI%vl?-XZU|`_9s^`&jD@&UQoZ?Es5Q+3{kV_ zfi)+snCB4G!tC~q7ei4Fom7fmXZhTOGucrYW9b6GoaeqGu}4{!AD1r|Rbiy|@j-@t zR{dTeja2QFcwNc65*m(7tjsTa`G*BQTLt}XC72cvgq&ol|PYo~Bi$L~( z^vrTu`w`H{EptCH>UB;oNkGMB(6nuM^)I%d>V}a0hb_QKzsCo~Pr&0cYG^O$%5Jll zlr#BEu-d<5{;e&@S)==I3%vIpYytDXY(WY;{b21&q*9b$ZNW?LArW70uq|j4uWg_O z&c?5^_EW~kx`dbO;h_!40LE_AU|&jIOiWR3Z;MJwW2d90I#7PTog~P@LoVq$8x0$Z zq=+!fnUwWIQ}&i6VC3Bbfz5tQ)~+1Ls)o_0{k_gB`vo#ezvSI*0!D$=Wh~`)5Y#FG zu(6(#L8Hqw3=Jho{k&YoehD7j{c|+FcTmG1 z|02Q{95X!UXk`(z|1qdEx3=IAS-ouYNNGmX(3R(BTfk4%U<_Y4SF>sF#@m=*vR+N~nIm<7 zK5AmSD6c5fG*WvH`Z#(roKTv!n%J$Y!DB~Wyht}))4@kI!8Sc%R2wAq-i(l|!~1#W z<^;l07(xxRIIA{$MD|WNtLj|0AGq&F$bI%L+nvq*gR*n4OXlsnR9rj!2uXR<5 zIeChPySPLbkuwh!M_lX*Yf5OO+d7H0#~-=v=1p>FyYtPk`)CBOZu_)d-CEHuX_mXLh7Proy7yI+6wD;NJLyt)5cIm^rcf8GQz+=JGHp&%gcp&=k- z{$Aiei1j#SE%|vyOt0lKf!AdyLpg7s$y-R~JP|~Ug{OEaYi+ND6p_j)hbVk?9FrPb zT`qYrbQ|$yG2#Zw;w4rLl;LL|0WsEqOBg9~62_^mUZ!A!_>}uJikOw24^Pq1ss(~G zvqbyG3}AE&V32__xlX{Wdpf->PGsg04L9K#yvCkZ46%Vp_)|ut+iaH8&-vR`A`vc^ z3TyqNyi8FlXv-aMc$#iGiH9iR(IiYLmq#ck_EFT)RvbeJrwV#ERj+0#ePE0iMoSAg z=<=na94I!_Vk)v~xZV#wGg0Z={Zd_H+nBqy^6hJ>o#LG1$zm99YA8}Vsk7EMg376u zz*&b|gRq5indLfxD$>aQPE5mAJ+%C|ld37uBR?tSq4ACi{!0mEs~4U_39PC~l4Fg` z6^o@v!B2J>I0k4TpaY-^VfSe zYehNlNC3-gAntA&mU_L2zZrX#p>O(nimyfw}} zowinjfZGWY(2bUW&$Q-TI*wGSMqq!bDPOvFxZFV+M+wbB4q`-e@9~u-=Dd$txU}$R z&H=w(7i1x;&d1yK6Z`=vEpy8sEQD3b-q0FIEkyNxVL5B_(C%!%LHOW;;Ec?TFBqAt zc$SWjSXDKnGkb3|;c*T4l?C6M)Ar<5B|`rsSbw80Jvg@nczpY*Gxf^bEMDZF-& z(Y<#>C9&N&kd(cvPanOBcJhemMT|H`)YfWz#4v%rLoY+|B{@Cj#1J0%rAz z4kU-{P^D5x?~^%Qg)bu_!=#cWv>-{>m`{nK0P9re(siI z5J=e)lW~!jn`GFS)TW}2U;1i&xchvb$wz2G_Y~S%VKlpFcL)n@^s;CTUpYfeO`@F# zi_L{wHg@oGF8*CrPZ`w=TUfC_X{5syLcag1uQYVh3q`{)Ov6W$tY+F@?+`;f4S3z{lH+(9zCcZQF%6r^leIi?KT<2dw zSC@~oL)_9vo3~784pkKA_RTy5$MpOb){A8??e5k1A{10HQ~z!AsQCxwNo&!tk7xEC zd*{!Fn4`?CV#?Hq?6g0dxGvB{1I8)_zA1ZwJM#lplW=A; z80w=1G7+KCO>jqXdTZR;XJ?w&@6Zw90}_baFXKy8?w;XXQd^ulZt1H-FjkYnSe?}Vj@A8La9sss zbpszGLo5M|)onHx=I}U~D==0o@*H9VnwFZ;q*#_jtua);^GP8k%RK?%)(lxJHg*UIu z`e^9J%ID;cUQA`1&T!x&E;z8ARS#=vA*A6uB8eXXwM;fhPKcOUd#G$WTJ(XpnQh}) zXB~QN7jFnlcu+@Q&jVIR-SfU<)s6@MHm`g9=)ei*=w4kV{SMsMgM`8a{P{8kuoM6f zG>AjNhwWc2TKMxg56c$5kB2xM;N3ru^FJbePso%5z=K1t<_MhMA{Ky^%_YY List[dict]: + """ + Raises: + Exception: If the segmentation of an annotation does not have 8 elements. + """ + + # validate segmentation first + for annot in coco_json["annotations"]: + validate_annotation(annot) + + log(message="creating BBoxLabelSet from COCO categories", level=logging.DEBUG) + bbox_label_set = DSBboxLabelSet( + id=None, + name="BBox Label Set", + classes=bbox_label_classes_from_coco(coco_json["categories"], custom_labelset), + ) + images = coco_json["images"] + + retval: List[dict] = [] + for image in images: + annotations_by_images = [ + annot + for annot in coco_json["annotations"] + if annot["image_id"] == image["id"] + ] + + bbox_labels = [ + bbox_label_from_coco_annotation( + annot, labelset=bbox_label_set, ignored_attributes=ignored_attributes + ) + for annot in annotations_by_images + ] + + schema = DatasaurSchema( + version="1", + data=DSBBoxProjectData( + kinds=["BBOX_BASED"], + bboxLabels=bbox_labels, + bboxLabelSets=[bbox_label_set], + document=GenericIdAndName(name=image["file_name"], id=None), + project=None, + pages=[ + DSPage( + pageIndex=0, + pageHeight=floor(image["height"]), + pageWidth=floor(image["width"]), + ) + ], + ), + ) + + retval.append(asdict(schema)) + + return retval + + +def main() -> None: + parser = ArgumentParser(prog="coco_to_datasaur_schemas") + parser.add_argument("coco_filepath", type=str, help="Path to COCO JSON file") + parser.add_argument( + "--custom-labelset", + type=str, + help="Path to custom labelset JSON file (useful for specifying DROPDOWN attributes)", + ) + parser.add_argument( + "--outdir", + type=str, + help="Output directory for Datasaur schemas", + default="./outdir/", + ) + parser.add_argument("--log-level", type=str, default="INFO") + parser.add_argument( + "--ignored-attributes", + type=str, + default="occluded", + help="Comma separated strings of attribute keys to ignore. Default: occluded", + ) + args = parser.parse_args() + logging.basicConfig(level=args.log_level, format="%(message)s") + + coco_filepath = os.path.abspath(args.coco_filepath) + log("reading COCO JSON file", filepath=coco_filepath) + with open(coco_filepath) as f: + json_data = json.load(f) + + custom_labelset = None + if args.custom_labelset: + custom_labelset_filepath = os.path.abspath(args.custom_labelset) + with open(custom_labelset_filepath) as f: + custom_labelset = json.load(f) + validate_bbox_labelset(custom_labelset) + + ignored_attributes = None + if args.ignored_attributes: + ignored_attributes = args.ignored_attributes.split(",") + + log("converting COCO to Datasaur Schema") + schemas = coco_to_datasaur_schemas( + json_data, + custom_labelset=custom_labelset, + ignored_attributes=ignored_attributes, + ) + + outdir = os.path.abspath(args.outdir) + os.makedirs(outdir, exist_ok=True) + log("writing to file", directory=outdir) + for schema in schemas: + image_filepath = schema["data"]["document"]["name"] + image_filename = os.path.basename(image_filepath) + filename = image_filename.split(".")[0] + ".json" + + with open(os.path.join(outdir, filename), "w") as f: + json.dump(scrub(schema), f, indent=2) + + +def bbox_label_classes_from_coco( + coco_categories: List[dict], + custom_labelset: Any | None, +) -> List[DSBBoxLabelClass]: + custom_classes = custom_labelset["classes"] if custom_labelset else [] + + retval = [] + for category in coco_categories: + # check and use questions from custom class + custom_class = next( + (item for item in custom_classes if item["name"] == category["name"]), None + ) + + questions = [ + DSBBoxLabelClassQuestions( + id=q.get("id", index), + label=q.get("label", f"Question {index}"), + config=QuestionConfig( + multiline=q.get("config", {}).get("multiline", None), + multiple=q.get("config", {}).get("multiple", None), + options=q.get("config", {}).get("options", None), + defaultValue=q.get("config", {}).get("defaultValue", None), + ), + required=q.get("required", False), + type=q.get("type", "TEXT"), + ) + for index, q in enumerate(defaults(custom_class, "questions", [])) + ] + + retval.append( + DSBBoxLabelClass( + id=str(category["id"]), + name=category["name"], + captionAllowed=defaults(custom_class, "captionAllowed", True), + captionRequired=defaults(custom_class, "captionRequired", False), + color=defaults(custom_class, "color", random_color(category["name"])), + questions=questions, + ) + ) + + return retval + + +def shape_from_coco_segmentation(segmentation: List[float]) -> DSShape: + points: List[DSPoint] = [] + + for i in range(0, 8, 2): + points.append(DSPoint(x=segmentation[i], y=segmentation[i + 1])) + + return DSShape(pageIndex=0, points=points) + + +def shape_from_coco_annotation(annotation: dict) -> DSShape: + segmentations_valid = all( + validate_segmentation(segment) for segment in annotation["segmentation"] + ) and (len(annotation["segmentation"]) > 0) + + if segmentations_valid: + return [ + shape_from_coco_segmentation(segment) + for segment in annotation["segmentation"] + ] + + log("found some invalid segmentations, using bbox instead", annotation=annotation) + x, y, width, height = annotation["bbox"] + return [ + DSShape( + pageIndex=0, + points=[ + DSPoint(x=x, y=y), + DSPoint(x=x + width, y=y), + DSPoint(x=x + width, y=y + height), + DSPoint(x=x, y=y + height), + ], + ) + ] + + +def bbox_label_from_coco_annotation( + annotation: dict, labelset: DSBboxLabelSet, ignored_attributes: list[str] | None +) -> DSBBoxLabel: + attributes = annotation["attributes"] + + stringified_id = str(annotation["category_id"]) + bbox_label_class = next( + item for item in labelset.classes if item.id == stringified_id + ) + + bbox_shapes = shape_from_coco_annotation(annotation) + + # check if attributes have any other key + # if found, add to labelset.classes + answers = None + ignored_attributes = set([*ignored_attributes, "text"]) + if attributes.keys() - ignored_attributes: + answers = {} + keys: Set = attributes.keys() - ignored_attributes + + for key in keys: + questions = bbox_label_class.questions + if questions is None: + questions = [] + + if key not in [q.label for q in questions]: + questions.append( + DSBBoxLabelClassQuestions( + id=len(questions), + label=key, + required=False, + type="TEXT", + config=QuestionConfig( + multiline=False, + multiple=False, + options=None, + defaultValue=None, + ), + ) + ) + + question_id = next(q.id for q in questions if q.label == key) + answers[str(question_id)] = str(attributes[key]) + + bbox_label_class.questions = questions + + return DSBBoxLabel( + id=str(annotation["id"]), + caption=str(attributes.get("text", "")), + bboxLabelClassId=bbox_label_class.id, + bboxLabelClassName=bbox_label_class.name, + shapes=bbox_shapes, + labeledBy=None, + acceptedByUserId=None, + labeledByUserId=None, + rejectedByUserId=None, + status=None, + answers=answers, + ) + + +if __name__ == "__main__": + main() diff --git a/bounding-boxes/src/common/defaults.py b/bounding-boxes/src/common/defaults.py new file mode 100644 index 0000000..eacf29f --- /dev/null +++ b/bounding-boxes/src/common/defaults.py @@ -0,0 +1,2 @@ +def defaults(obj: dict | None, key: str, default): + return obj[key] if obj and key in obj else default diff --git a/bounding-boxes/src/common/logger.py b/bounding-boxes/src/common/logger.py new file mode 100644 index 0000000..183bf09 --- /dev/null +++ b/bounding-boxes/src/common/logger.py @@ -0,0 +1,23 @@ +# https://docs.python.org/3/howto/logging-cookbook.html#implementing-structured-logging +import json +import logging + + +class StructuredMessage: + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + + def __str__(self) -> str: + return json.dumps(self.kwargs) + + +def log(message, logger: logging.Logger, level=logging.INFO, **kwargs): + logger.log( + level=level, + msg=StructuredMessage( + level=logging._levelToName[level], + message=message, + **kwargs, + logger=logger.name, + ), + ) diff --git a/bounding-boxes/src/common/random_color.py b/bounding-boxes/src/common/random_color.py new file mode 100644 index 0000000..77aa557 --- /dev/null +++ b/bounding-boxes/src/common/random_color.py @@ -0,0 +1,7 @@ +import random + + +def random_color(seed: str) -> str: + random.seed(seed) + r = lambda: random.randint(0, 255) + return "#%02X%02X%02X" % (r(), r(), r()) diff --git a/bounding-boxes/src/common/scrub.py b/bounding-boxes/src/common/scrub.py new file mode 100644 index 0000000..540a012 --- /dev/null +++ b/bounding-boxes/src/common/scrub.py @@ -0,0 +1,24 @@ +import copy + + +def scrub(x): + """ + Adapted from https://stackoverflow.com/a/11410935 + """ + ret = copy.deepcopy(x) + if isinstance(x, dict): + for key, value in x.items(): + if isinstance(value, dict): + ret[key] = scrub(value) + continue + + if value is None: + del ret[key] + continue + + ret[key] = scrub(value) + elif isinstance(x, list): + for index, item in enumerate(x): + ret[index] = scrub(item) + + return ret diff --git a/bounding-boxes/src/datasaur_schemas_to_coco.py b/bounding-boxes/src/datasaur_schemas_to_coco.py new file mode 100644 index 0000000..1f3d42e --- /dev/null +++ b/bounding-boxes/src/datasaur_schemas_to_coco.py @@ -0,0 +1,276 @@ +import json +import os +from argparse import ArgumentParser +from dataclasses import asdict +from shutil import rmtree +from typing import Any, Dict +from zipfile import Path as ZipPath +from zipfile import ZipFile +import logging + + +from common.logger import log as _log +from formats.coco import COCO, COCOAnnotation, COCOCategory, COCOImage + + +def log(message, level=logging.DEBUG, **kwargs): + logger = logging.getLogger( + __name__ if __name__ != "__main__" else "datasaur_schemas_to_coco" + ) + return _log(message=message, logger=logger, level=level, **kwargs) + + +def datasaur_schemas_to_coco( + schema_objects: list[Any], + licenses: list[dict] | None = None, + info: dict | None = None, +) -> dict: + """ + Convert Datasaur schema objects to COCO format. + + Args: + schema_objects (list[Any]): A list of dictionaries representing the result of `json.load`-ing the Datasaur Schema JSON. + licenses (list[COCOLicense] | None, optional): A list of COCOLicense objects. Defaults to None. + info (COCOInfo | None, optional): A COCOInfo object. Defaults to None. + + Returns: + dict: A dictionary representing the COCO format. + + """ + if licenses is None: + licenses = [{"name": "dummy-datasaur-license", "id": 0, "url": ""}] + + if info is None: + info = { + "contributor": "Datasaur", + "date_created": "2024-04-23", + "description": "Exported from Datasaur", + "url": "https://datasaur.ai", + "version": "v0.1", + "year": 2024, + } + + schemas = [s for s in schema_objects] + + # assuming all DatasaurSchema are from the same project, + # there will be the same bboxLabelSet + categories: list[COCOCategory] = coco_categories_from_datasaur_schema(schemas[0]) + + # images from datasaur schemas -- name, dimension info + images: list[COCOImage] = [ + coco_images_from_datasaur_schema(id, schema) + for id, schema in enumerate(schemas) + ] + + # annotations from bboxLabels + annotations: list[COCOAnnotation] = coco_annots_from_datasaur_schemas( + schemas, categories + ) + + return asdict( + COCO( + info=info, + licenses=licenses, + categories=categories, + images=images, + annotations=annotations, + ) + ) + + +def main() -> None: + parser = ArgumentParser(prog="datasaur_schemas_to_coco") + parser.add_argument( + "zip_filepath", type=str, help="Path to Datasaur export ZIP file" + ) + parser.add_argument( + "--outfile", + type=str, + help="Output directory for COCO JSON file", + default="./outdir/coco.json", + ) + parser.add_argument( + "--license-and-info-json", + type=str, + help="Path to JSON file containing licenses and info data", + default="samples/license-and-info.json", + ) + parser.add_argument("--log-level", type=str, default="INFO") + args = parser.parse_args() + logging.basicConfig(level=args.log_level, format="%(message)s") + + export_zip = os.path.abspath(args.zip_filepath) + temp_destination = os.path.abspath("./temp/") + os.makedirs(temp_destination, exist_ok=True) + log("creating temp directory", directory=temp_destination) + + outfile = os.path.abspath(args.outfile) + outdir = os.path.dirname(outfile) + os.makedirs(outdir, exist_ok=True) + + extracted_files: list[str] = unzip_export_result( + export_zip=export_zip, dest=temp_destination + ) + + schemas = [load_datasaur_schema_file(f) for f in extracted_files] + + log("reading licenses and info file", filepath=args.license_and_info_json) + license_and_info = json.load(open(os.path.abspath(args.license_and_info_json))) + + log("converting datasaur schemas to COCO format", count=len(schemas)) + coco = datasaur_schemas_to_coco( + schemas, + licenses=license_and_info.get("licenses", None), + info=license_and_info.get("info", None), + ) + + log("writing COCO JSON file", outfile=outfile) + with open(outfile, "w") as wf: + json.dump(coco, wf, indent=2) + + log("cleaning up temp directory", directory=temp_destination) + rmtree(temp_destination) + + +def coco_annots_from_datasaur_schemas( + schemas: list[dict], categories: list[COCOCategory] +) -> list[COCOAnnotation]: + + name_to_id: Dict[str, int] = {x.name: x.id for x in categories} + annots: list[COCOAnnotation] = [] + for image_id, schema in enumerate(schemas): + annot_id = 1 + + if ( + schema["data"]["bboxLabelSets"] is None + or len(schema["data"]["bboxLabels"]) < 1 + ): + continue + + labelset = schema["data"]["bboxLabelSets"][0] + label_id_to_question_info = { + label["id"]: {str(q.get("id")): q for q in label.get("questions", {})} + for label in labelset["classes"] + } + + if schema["data"]["bboxLabels"] is None: + continue + + for bbox_label in schema["data"]["bboxLabels"]: + answers = bbox_label.get("answers", None) + attributes = {} + if answers: + questions = label_id_to_question_info.get( + bbox_label["bboxLabelClassId"], None + ) + + for key, value in answers.items(): + question_label = questions[key]["label"] + attributes[question_label] = value + + annots.append( + COCOAnnotation( + id=annot_id, + image_id=image_id, + category_id=name_to_id[bbox_label["bboxLabelClassName"]], + segmentation=shapes_to_segmentation(bbox_label["shapes"]), + bbox=shapes_to_bbox(bbox_label["shapes"]), + attributes={"text": bbox_label.get("caption", None), **attributes}, + area=0, + iscrowd=0, + ) + ) + + return annots + + +def coco_categories_from_datasaur_schema(schema: dict) -> list[COCOCategory]: + if ( + schema["data"]["bboxLabelSets"] is None + or len(schema["data"]["bboxLabelSets"]) < 1 + ): + return [] + + retval: list[COCOCategory] = [] + + # Currently, there can only be 1 bboxLabelSets + bbox_label_set = schema["data"]["bboxLabelSets"][0] + for i, label_class in enumerate(bbox_label_set["classes"], 1): + retval.append(COCOCategory(id=i, name=label_class["name"], supercategory="")) + + return retval + + +def coco_images_from_datasaur_schema(id: int, schema: dict) -> COCOImage: + width, height = 0, 0 + if schema["data"]["pages"] is not None and len(schema["data"]["pages"]) >= 1: + width = schema["data"]["pages"][0]["pageWidth"] + height = schema["data"]["pages"][0]["pageHeight"] + + return COCOImage( + id=id, + file_name=schema["data"]["document"]["name"], + width=width, + height=height, + license=0, + flickr_url="", + coco_url="", + ) + + +def shapes_to_bbox(shapes: list[dict]) -> list[float]: + x_coords = [point["x"] for shape in shapes for point in shape["points"]] + y_coords = [point["y"] for shape in shapes for point in shape["points"]] + + x_min = min(*x_coords) + x_max = max(*x_coords) + y_min, y_max = min(*y_coords), max(*y_coords) + + return [min(*x_coords), min(*y_coords), x_max - x_min, y_max - y_min] + + +def shapes_to_segmentation(shapes: list[dict]) -> list[list[float]]: + retval: list[list[float]] = [] + for shape in shapes: + segmentation: list[float] = [ + dot for point in shape["points"] for dot in [point["x"], point["y"]] + ] + retval.append(segmentation) + return retval + + +def unzip_export_result(export_zip: str, dest: str) -> list[str]: + retval: list[str] = [] + log("unzipping export result to temp directory", export_zip=export_zip) + with ZipFile(export_zip, "r") as zf: + project_dir: str | None = None + for zippath in ZipPath(zf).iterdir(): + if zippath.is_dir(): + project_dir = zippath.name + break + if not (project_dir): + log("no project dir found in export result", level=logging.ERROR) + raise Exception("no project dir found") + + log("project_dir", project_dir=project_dir) + for zip_content in zf.infolist(): + if not zip_content.filename.startswith(os.path.join(project_dir, "REVIEW")): + continue + + if zip_content.is_dir(): + zf.extract(zip_content.filename, dest) + continue + retval.append(zf.extract(zip_content, dest)) + + return retval + + +def load_datasaur_schema_file(filepath: str): + with open(filepath) as f: + data = json.load(f) + + return data + + +if __name__ == "__main__": + main() diff --git a/bounding-boxes/src/formats/bbox_labelset.py b/bounding-boxes/src/formats/bbox_labelset.py new file mode 100644 index 0000000..5d8b00f --- /dev/null +++ b/bounding-boxes/src/formats/bbox_labelset.py @@ -0,0 +1,81 @@ +LABELSET_KEYS = {"classes", "name", "autoLabelProvider"} +CLASS_KEYS = {"id", "name", "captionAllowed", "captionRequired", "color", "questions"} +QUESTION_KEYS = {"id", "label", "required", "type", "config", "internalId"} +DROPDOWN_KEYS = {"defaultValue", "multiple", "options"} +UNUSED_TEXT_KEYS = {"minLength", "pattern", "maxLength"} +TEXT_KEYS = {"defaultValue", "multiline", "multiple", *UNUSED_TEXT_KEYS} +OPTION_KEYS = {"id", "label", "parentId"} + + +def validate_bbox_labelset(bbox_labelset: dict): + assertKeys(bbox_labelset, LABELSET_KEYS, "labelset object") + if "classes" not in bbox_labelset: + raise AssertionError("expected classes in labelset") + + for index, labelclass in enumerate(bbox_labelset["classes"]): + assertKeys(labelclass, CLASS_KEYS, f"labelclass: {index}") + if "name" not in labelclass: + raise AssertionError("expected name in labelclass") + + for question in labelclass.get("questions", []): + assertExact( + question, + QUESTION_KEYS, + f"question of label: {labelclass['name']}", + ) + + if question["type"] == "DROPDOWN": + assertDropdownQuestion(question) + + if question["type"] == "TEXT": + assertKeys( + question["config"], + TEXT_KEYS, + f"config of question: {question['label']}", + ) + + +def assertDropdownQuestion(question: dict): + assertKeys( + question["config"], + DROPDOWN_KEYS, + f"config of question {question['label']}", + ) + + if not isinstance(question["config"]["options"], list): + raise AssertionError( + f"expects options as a list, received {type(question['config']['options'])}" + ) + + for opt in question["config"]["options"]: + assertKeys( + opt, + OPTION_KEYS, + f"options of question {question['label']}", + ) + + +def assertKeys(dict: dict, valid_keys: set, identifier: str | None = None): + """ + Ensure that dict keys are subset of valid_keys + """ + keys = set(dict.keys()) + if not keys.issubset(valid_keys): + raise AssertionError( + f"Invalid keys detected in {identifier}: {keys.difference(valid_keys)}" + ) + + +def assertExact(dict: dict, valid_keys: set, identifier: str | None = None): + """ + Ensure that dict keys are exactly equal to valid_keys + """ + keys = set(dict.keys()) + if not keys == valid_keys: + if keys.issubset(valid_keys): + raise AssertionError( + f"Missing keys detected in {identifier}: {valid_keys.difference(keys)}" + ) + raise AssertionError( + f"Excess keys detected in {identifier}: {keys.difference(valid_keys)}" + ) diff --git a/bounding-boxes/src/formats/coco.py b/bounding-boxes/src/formats/coco.py new file mode 100644 index 0000000..0591e5c --- /dev/null +++ b/bounding-boxes/src/formats/coco.py @@ -0,0 +1,98 @@ +from dataclasses import dataclass +from typing import Dict, List + +SUPPORTED_SEGMENTATION_LENGTH = 8 + + +@dataclass +class COCOLicense: + name: str + id: int + url: str + + +@dataclass +class COCOCategory: + id: int + name: str + supercategory: str + + +@dataclass +class COCOImage: + id: int + width: float + height: float + file_name: str + license: int + flickr_url: str + coco_url: str + + +@dataclass +class COCOAnnotation: + id: int + image_id: int + category_id: int + """ + Expected a list of 8 numbers, representing four vertices starting from top-left and going clockwise + """ + segmentation: List[List[float]] + + """ + [x, y, width, height] + """ + bbox: List[float] + area: float + """ + int, but mainly 0/1 + """ + iscrowd: int + attributes: Dict[str, str | int | bool] + + +@dataclass +class COCOInfo: + contributor: str + date_created: str + description: str + url: str + version: str + year: int | str + + +@dataclass +class COCOForInput: + categories: List[COCOCategory] + images: List[COCOImage] + annotations: List[COCOAnnotation] + + +@dataclass +class COCO(COCOForInput): + licenses: List[COCOLicense] + info: COCOInfo + + +def validate_annotation(annotation): + """ + Ensure that the segmentation of an annotation has 8 elements, + or failing that, ensure it has a valid bounding boxes + """ + bbox = annotation["bbox"] + + shoud_throw = len(bbox) != 4 + + if shoud_throw: + for segmentation in annotation["segmentation"]: + if not validate_segmentation(segmentation): + raise Exception( + "expect segmentation to be a list-of-list of 8 elements" + ) + + +def validate_segmentation(segmentation): + if len(segmentation) != SUPPORTED_SEGMENTATION_LENGTH: + return False + + return True diff --git a/bounding-boxes/src/formats/datasaur_schema.py b/bounding-boxes/src/formats/datasaur_schema.py new file mode 100644 index 0000000..b84deaa --- /dev/null +++ b/bounding-boxes/src/formats/datasaur_schema.py @@ -0,0 +1,98 @@ +from dataclasses import dataclass +from typing import List, Optional, Dict + + +@dataclass +class GenericIdAndName: + id: Optional[str] + name: str + + +@dataclass +class DSPoint: + x: float + y: float + + +@dataclass +class DSShape: + pageIndex: int + points: List[DSPoint] + + +@dataclass +class DSBBoxLabel: + id: str + bboxLabelClassId: str + bboxLabelClassName: str + caption: Optional[str] + shapes: List[DSShape] + """ + tba + """ + answers: Optional[Dict[str, str | int | bool]] + + # For Datasaur Schema used in input, we can ignore these + status: Optional[str] + labeledBy: Optional[str] + labeledByUserId: Optional[int] + acceptedByUserId: Optional[int] + rejectedByUserId: Optional[int] + + +@dataclass +class QuestionConfig: + multiline: Optional[bool] + multiple: Optional[bool] + options: Optional[List[dict[str, str]]] + defaultValue: Optional[str] + + +@dataclass +class DSBBoxLabelClassQuestions: + id: int + label: str + required: bool + type: str + config: QuestionConfig + pass + + +@dataclass +class DSBBoxLabelClass(GenericIdAndName): + id: str + color: Optional[str] + captionAllowed: bool + captionRequired: bool + """ + tba + """ + questions: Optional[List[DSBBoxLabelClassQuestions]] + + +@dataclass +class DSBboxLabelSet(GenericIdAndName): + classes: List[DSBBoxLabelClass] + + +@dataclass +class DSPage: + pageIndex: int + pageHeight: int + pageWidth: int + + +@dataclass +class DSBBoxProjectData: + kinds: List[str] + pages: Optional[List[DSPage]] + bboxLabelSets: Optional[List[DSBboxLabelSet]] + bboxLabels: Optional[List[DSBBoxLabel]] + document: GenericIdAndName + project: Optional[GenericIdAndName] + + +@dataclass +class DatasaurSchema: + data: DSBBoxProjectData + version: str diff --git a/create-project-async/api_client.py b/create-project-async/api_client.py index b541cc1..825cc56 100644 --- a/create-project-async/api_client.py +++ b/create-project-async/api_client.py @@ -1,11 +1,27 @@ +import logging from os import environ +import traceback import fire +from src.helper import parse_multiple_config from src.job import Job +from src.logger import log as log from src.project import Project -def create_project(base_url, client_id, client_secret, team_id, documents_path="documents", operations_path="create_project.json"): +def logError(message, level=logging.ERROR, **kwargs): + logger = logging.getLogger("api_client") + return log(logger=logger, level=level, message=message, **kwargs) + + +def create_project( + base_url, + client_id, + client_secret, + team_id, + documents_path="documents", + operations_path="create_project.json", +): try: Project(base_url=base_url, id=client_id, secret=client_secret).create( team_id=team_id, @@ -29,6 +45,32 @@ def get_job_status(base_url, client_id, client_secret, job_id): raise SystemExit(e) +def create_multiple_projects( + base_url, + client_id, + client_secret, + team_id, + operations_path="create_project.json", + config="config.csv", +): + project_configs = parse_multiple_config(config) + for name, documents_path in project_configs: + try: + Project(base_url=base_url, id=client_id, secret=client_secret).create( + team_id=team_id, + documents_path=documents_path, + operations_path=operations_path, + name=name, + ) + except Exception as e: + logError( + message=f"Error creating project: {name}", + exception=traceback.format_exception(e), + ) + + if __name__ == "__main__": environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" + logging.basicConfig(level=logging.INFO, format="%(message)s") + fire.Fire() diff --git a/create-project-async/config.csv b/create-project-async/config.csv new file mode 100644 index 0000000..c3f2457 --- /dev/null +++ b/create-project-async/config.csv @@ -0,0 +1,2 @@ +"project 1","./documents" +"project 2","./documents" diff --git a/create-project-async/readme.md b/create-project-async/readme.md index f88d50f..46e50c5 100644 --- a/create-project-async/readme.md +++ b/create-project-async/readme.md @@ -30,6 +30,26 @@ python api_client.py create_project \ --team_id TEAM_ID ``` +## Create Multiple Projects + +A thin wrapper around the logic for `create_project`. +`CONFIG_FILE` should be a CSV file without header, where the first column contains project names, while the second column contains path to the folders. + +```csv +"project1","./documents" +"project2","./documents" +``` + +``` +python api_client.py create_multiple_projects \ + --base_url BASE_URL \ + --client_id CLIENT_ID \ + --client_secret CLIENT_SECRET \ + --team_id TEAM_ID + --config CONFIG_FILE +``` + + ## Get Job Status ``` @@ -38,4 +58,4 @@ python api_client.py get_job_status \ --client_id CLIENT_ID \ --client_secret CLIENT_SECRET \ --job_id JOB_ID -``` +``` \ No newline at end of file diff --git a/create-project-async/src/helper.py b/create-project-async/src/helper.py index 1fcee73..2f716b8 100644 --- a/create-project-async/src/helper.py +++ b/create-project-async/src/helper.py @@ -1,16 +1,26 @@ from oauthlib.oauth2 import BackendApplicationClient from requests_oauthlib import OAuth2Session import json +from csv import reader as csvreader def get_access_token(base_url, client_id, client_secret): client = BackendApplicationClient(client_id=client_id) oauth = OAuth2Session(client=client) - token = oauth.fetch_token(token_url=base_url + '/api/oauth/token', - client_id=client_id, client_secret=client_secret) - return token['access_token'] + token = oauth.fetch_token( + token_url=base_url + "/api/oauth/token", + client_id=client_id, + client_secret=client_secret, + ) + return token["access_token"] def get_operations(file_name): - with open(file_name, 'r') as file: + with open(file_name, "r") as file: return json.loads(file.read()) + + +def parse_multiple_config(config_path: str): + with open(config_path, "r") as file: + config_reader = csvreader(file) + yield from config_reader diff --git a/create-project-async/src/logger.py b/create-project-async/src/logger.py new file mode 100644 index 0000000..183bf09 --- /dev/null +++ b/create-project-async/src/logger.py @@ -0,0 +1,23 @@ +# https://docs.python.org/3/howto/logging-cookbook.html#implementing-structured-logging +import json +import logging + + +class StructuredMessage: + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + + def __str__(self) -> str: + return json.dumps(self.kwargs) + + +def log(message, logger: logging.Logger, level=logging.INFO, **kwargs): + logger.log( + level=level, + msg=StructuredMessage( + level=logging._levelToName[level], + message=message, + **kwargs, + logger=logger.name, + ), + ) diff --git a/create-project-async/src/project.py b/create-project-async/src/project.py index 11d7d3e..8a19c67 100644 --- a/create-project-async/src/project.py +++ b/create-project-async/src/project.py @@ -7,7 +7,7 @@ class Project: - def __init__(self, base_url: str, id:str, secret:str): + def __init__(self, base_url: str, id: str, secret: str): self.base_url = base_url self.graphql_url = f"{base_url}/graphql" self.proxy_url = f"{base_url}/api/static/proxy/upload" @@ -15,12 +15,18 @@ def __init__(self, base_url: str, id:str, secret:str): self.client_secret = secret self.headers = None - def create(self, team_id, operations_path, documents_path): - if (os.path.isfile(documents_path)): - raise NotImplementedError("createProject with a list of documents is not yet implemented") + def create(self, team_id, operations_path, documents_path, name=None): + if os.path.isfile(documents_path): + raise NotImplementedError( + "createProject with a list of documents is not yet implemented" + ) - access_token = get_access_token(self.base_url, self.client_id, self.client_secret) - self.headers = self.__add_headers(key='Authorization', value=f"Bearer {access_token}") + access_token = get_access_token( + self.base_url, self.client_id, self.client_secret + ) + self.headers = self.__add_headers( + key="Authorization", value=f"Bearer {access_token}" + ) operations = get_operations(operations_path) operations["variables"]["input"]["documents"] = [] @@ -31,40 +37,50 @@ def create(self, team_id, operations_path, documents_path): mapped_documents = self.__map_documents(sorted_filepaths) for key in mapped_documents: - upload_document_response = self.__upload_file(filepath=mapped_documents[key]["document"]) + upload_document_response = self.__upload_file( + filepath=mapped_documents[key]["document"] + ) documents = { "document": { "name": os.path.basename(mapped_documents[key]["document"]), - "objectKey": upload_document_response["objectKey"] + "objectKey": upload_document_response["objectKey"], } } if "extra" in mapped_documents[key]: - upload_extra_response = self.__upload_file(filepath=mapped_documents[key]["extra"]) + upload_extra_response = self.__upload_file( + filepath=mapped_documents[key]["extra"] + ) documents["extras"] = [ { "name": os.path.basename(mapped_documents[key]["extra"]), - "objectKey": upload_extra_response["objectKey"] + "objectKey": upload_extra_response["objectKey"], } ] operations["variables"]["input"]["documents"].append(documents) + if name is not None: + operations["variables"]["input"]["name"] = name + graphql_response = self.__call_graphql( data={ "query": operations["query"], "variables": json.dumps(operations["variables"]), - "operationName": operations.get("operationName", "Datasaur API client - createProject") + "operationName": operations.get( + "operationName", "Datasaur API client - createProject" + ), } ) + self.__process_graphql_response(graphql_response) - + def __upload_file(self, filepath): with post( - url=self.proxy_url, - headers=self.headers, - files=[('file', open(filepath, 'rb'))] - ) as response: + url=self.proxy_url, + headers=self.headers, + files=[("file", open(filepath, "rb"))], + ) as response: response.raise_for_status() return response.json() @@ -85,10 +101,10 @@ def __process_graphql_response(self, response): else: print(response.text.encode("utf8")) print(response) - + def __add_headers(self, key, value): if self.headers is None: - self.headers = {key: value} + self.headers = {key: value} else: self.headers[key] = value @@ -96,15 +112,15 @@ def __add_headers(self, key, value): def __sort_possible_extra_files_last(self, filepaths): # Sort file paths ending with .json or .txt to be at the end - filepaths.sort(key=lambda x: (x.endswith('.json') or x.endswith('.txt'), x)) + filepaths.sort(key=lambda x: (x.endswith(".json") or x.endswith(".txt"), x)) return filepaths def __map_documents(self, filepaths): mapped_documents = {} - for filepath in filepaths: - filename = os.path.basename(filepath).split('.')[0] - if filename in mapped_documents: + for filepath in filepaths: + filename = os.path.basename(filepath).split(".")[0] + if filename in mapped_documents: mapped_documents[filename]["extra"] = filepath - else: - mapped_documents[filename] = { "document": filepath } + else: + mapped_documents[filename] = {"document": filepath} return mapped_documents From 534ee6902f226ece5332b1ff35bd6b5491d1bd44 Mon Sep 17 00:00:00 2001 From: Ivan Marcellino Date: Tue, 25 Jun 2024 12:18:03 +0700 Subject: [PATCH 2/7] use splittext to obtain original filename (#32) to handle filenames with multiple dots. also: - fix: typings for shapes_from_coco - fix: adjust None handling for ignored_attributes --- .../src/coco_to_datasaur_schemas.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/bounding-boxes/src/coco_to_datasaur_schemas.py b/bounding-boxes/src/coco_to_datasaur_schemas.py index d6737eb..431e35b 100644 --- a/bounding-boxes/src/coco_to_datasaur_schemas.py +++ b/bounding-boxes/src/coco_to_datasaur_schemas.py @@ -64,7 +64,9 @@ def coco_to_datasaur_schemas( bbox_labels = [ bbox_label_from_coco_annotation( - annot, labelset=bbox_label_set, ignored_attributes=ignored_attributes + annot, + labelset=bbox_label_set, + ignored_attributes=ignored_attributes, ) for annot in annotations_by_images ] @@ -145,9 +147,10 @@ def main() -> None: for schema in schemas: image_filepath = schema["data"]["document"]["name"] image_filename = os.path.basename(image_filepath) - filename = image_filename.split(".")[0] + ".json" + filename, _extension = os.path.splitext(image_filename) + answer_filename = filename + ".json" - with open(os.path.join(outdir, filename), "w") as f: + with open(os.path.join(outdir, answer_filename), "w") as f: json.dump(scrub(schema), f, indent=2) @@ -203,7 +206,7 @@ def shape_from_coco_segmentation(segmentation: List[float]) -> DSShape: return DSShape(pageIndex=0, points=points) -def shape_from_coco_annotation(annotation: dict) -> DSShape: +def shape_from_coco_annotation(annotation: dict) -> list[DSShape]: segmentations_valid = all( validate_segmentation(segment) for segment in annotation["segmentation"] ) and (len(annotation["segmentation"]) > 0) @@ -232,6 +235,9 @@ def shape_from_coco_annotation(annotation: dict) -> DSShape: def bbox_label_from_coco_annotation( annotation: dict, labelset: DSBboxLabelSet, ignored_attributes: list[str] | None ) -> DSBBoxLabel: + if ignored_attributes is None: + ignored_attributes = [] + attributes = annotation["attributes"] stringified_id = str(annotation["category_id"]) @@ -244,10 +250,10 @@ def bbox_label_from_coco_annotation( # check if attributes have any other key # if found, add to labelset.classes answers = None - ignored_attributes = set([*ignored_attributes, "text"]) - if attributes.keys() - ignored_attributes: + set_ignored_attributes = set([*ignored_attributes, "text"]) + if attributes.keys() - set_ignored_attributes: answers = {} - keys: Set = attributes.keys() - ignored_attributes + keys: Set = attributes.keys() - set_ignored_attributes for key in keys: questions = bbox_label_class.questions From 0e9eef1ff26dc5b6998f662218c158123e0d70e9 Mon Sep 17 00:00:00 2001 From: Abel <96862436+abelrischa@users.noreply.github.com> Date: Sat, 6 Jul 2024 22:10:07 +0700 Subject: [PATCH 3/7] batched project creation (#31) * separated some logics from Project class to new classes GraphQLUtils and GraphQLDocumentCreator * createdf ProjectInBatch class to create multiple projects based on the amount of batched documents * updated api_client for the new usage of ProjectInBatch and Project classes * Fix code smells in api_client.py, reorder imports * add type hints to object initialization and method params, change self.headers default value to empty dict * update readme to mention minimum Python version * change default document_batch_size to 20 * change private methods to protected * change default batch size to 100 * rename ProjectInBatch to BatchedProject * fix typo in log message * introduce __report_before_request * update readme, add sub-section about batched project creation * extracted default documents path to constant * moved getting access token for Header to Project.__init__ * import GraphQLDocument from graphql_document_creator * added logging when getting access token and uploading files * remove unused import * use ThreadPoolExecutor to upload multiple files at once * add logging: total project creation request that will be submitted * move upload_and_create_document to a class method * #23456 add xml to __sort_possible_extra_files_last * #23456 remove print * add xml to __sort_possible_extra_files_last * use `sorted` and extract valid extensions as a tuple --------- Co-authored-by: Ivan Marcellino --- create-project-async/api_client.py | 42 ++++-- create-project-async/readme.md | 34 +++-- create-project-async/src/batched_project.py | 64 +++++++++ .../src/graphql_document_creator.py | 103 +++++++++++++++ create-project-async/src/graphql_utils.py | 30 +++++ create-project-async/src/helper.py | 2 + create-project-async/src/project.py | 122 ++++-------------- 7 files changed, 283 insertions(+), 114 deletions(-) create mode 100644 create-project-async/src/batched_project.py create mode 100644 create-project-async/src/graphql_document_creator.py create mode 100644 create-project-async/src/graphql_utils.py diff --git a/create-project-async/api_client.py b/create-project-async/api_client.py index 825cc56..bfbebf5 100644 --- a/create-project-async/api_client.py +++ b/create-project-async/api_client.py @@ -1,15 +1,19 @@ import logging -from os import environ import traceback - import fire + +from os import environ +from src.batched_project import DEFAULT_BATCH_SIZE, BatchedProject from src.helper import parse_multiple_config from src.job import Job from src.logger import log as log from src.project import Project +DEFAULT_OPERATIONS_PATH = "create_project.json" +DEFAULT_DOCUMENTS_PATH = "documents" + -def logError(message, level=logging.ERROR, **kwargs): +def log_error(message, level=logging.ERROR, **kwargs): logger = logging.getLogger("api_client") return log(logger=logger, level=level, message=message, **kwargs) @@ -19,13 +23,30 @@ def create_project( client_id, client_secret, team_id, - documents_path="documents", - operations_path="create_project.json", + documents_path=DEFAULT_DOCUMENTS_PATH, + operations_path=DEFAULT_OPERATIONS_PATH, +): + try: + Project(base_url=base_url, id=client_id, secret=client_secret, documents_path=documents_path).create( + team_id=team_id, + operations_path=operations_path, + ) + except Exception as e: + raise SystemExit(e) + + +def create_batched_projects( + base_url, + client_id, + client_secret, + team_id, + documents_path=DEFAULT_DOCUMENTS_PATH, + operations_path=DEFAULT_OPERATIONS_PATH, + document_batch_size=DEFAULT_BATCH_SIZE, ): try: - Project(base_url=base_url, id=client_id, secret=client_secret).create( + BatchedProject(base_url=base_url, id=client_id, secret=client_secret, documents_path=documents_path, document_batch_size=document_batch_size).create( team_id=team_id, - documents_path=documents_path, operations_path=operations_path, ) except Exception as e: @@ -50,20 +71,19 @@ def create_multiple_projects( client_id, client_secret, team_id, - operations_path="create_project.json", + operations_path=DEFAULT_OPERATIONS_PATH, config="config.csv", ): project_configs = parse_multiple_config(config) for name, documents_path in project_configs: try: - Project(base_url=base_url, id=client_id, secret=client_secret).create( + Project(base_url=base_url, id=client_id, secret=client_secret, documents_path=documents_path).create( team_id=team_id, - documents_path=documents_path, operations_path=operations_path, name=name, ) except Exception as e: - logError( + log_error( message=f"Error creating project: {name}", exception=traceback.format_exception(e), ) diff --git a/create-project-async/readme.md b/create-project-async/readme.md index 46e50c5..788ccd0 100644 --- a/create-project-async/readme.md +++ b/create-project-async/readme.md @@ -2,10 +2,12 @@ ## Pre-requisite -``` -# install dependencies -python -m pip install -r src/requirements.txt -``` +- Python 3.10 or higher +- Install dependencies + ``` + # install dependencies + python -m pip install -r src/requirements.txt + ``` ## Create Project (v2) @@ -17,7 +19,7 @@ In this new mutation, we no longer support uploading files directly to the Graph ### With Local Files -Local files are located under `document/` folder. +Local files are located under `documents/` folder. Every file inside the directory will be uploaded to Datasaur as part of the project creation process. **Note**: If you want to use pairing files, such as inputfile.jpg with inputfile.txt, ensure that they share the same filename. These paired files are commonly used for OCR / Audio projects with transcription. @@ -30,9 +32,24 @@ python api_client.py create_project \ --team_id TEAM_ID ``` +### Batched Project Creation + +A wrapper around the logic for `create_project`. + +Requires an additional `--document_batch_size` param. The value of this param determines the max amount of documents in each project. The batch size value must be between 1 and 100. + +``` +python api_client.py create_batched_projects \ + --base_url BASE_URL \ + --client_id CLIENT_ID \ + --client_secret CLIENT_SECRET \ + --team_id TEAM_ID \ + --document_batch_size 100 +``` + ## Create Multiple Projects -A thin wrapper around the logic for `create_project`. +A thin wrapper around the logic for `create_project`. `CONFIG_FILE` should be a CSV file without header, where the first column contains project names, while the second column contains path to the folders. ```csv @@ -45,11 +62,10 @@ python api_client.py create_multiple_projects \ --base_url BASE_URL \ --client_id CLIENT_ID \ --client_secret CLIENT_SECRET \ - --team_id TEAM_ID + --team_id TEAM_ID --config CONFIG_FILE ``` - ## Get Job Status ``` @@ -58,4 +74,4 @@ python api_client.py get_job_status \ --client_id CLIENT_ID \ --client_secret CLIENT_SECRET \ --job_id JOB_ID -``` \ No newline at end of file +``` diff --git a/create-project-async/src/batched_project.py b/create-project-async/src/batched_project.py new file mode 100644 index 0000000..8dc36d4 --- /dev/null +++ b/create-project-async/src/batched_project.py @@ -0,0 +1,64 @@ +import json +from src.graphql_document_creator import GraphQLDocumentCreator +from src.graphql_utils import GraphQLUtils +from src.project import Project + +DEFAULT_BATCH_SIZE = 100 + + +class BatchedProject(Project): + def __init__(self, base_url: str, id: str, secret: str, documents_path: str, document_batch_size=DEFAULT_BATCH_SIZE): + if not 1 <= document_batch_size <= 100: + raise ValueError("document_batch_size must be between 1 and 100") + + super().__init__(base_url, id, secret, documents_path) + self.document_batch_size = document_batch_size + self.graphql_utils = GraphQLUtils(base_url=self.base_url, headers=self.headers, + client_id=self.client_id, client_secret=self.client_secret) + + def create(self, team_id: str, operations_path: str, name: str | None = None): + chunked_gql_documents = self.__get_chunked_gql_documents() + print(f"creating {len(chunked_gql_documents)} projects...") + for index, gql_documents in enumerate(chunked_gql_documents): + operations = self._get_operations( + team_id, operations_path, gql_documents, name) + + name_with_batch_number = self.__get_name_with_batch_number( + operations["variables"]["input"]["name"], index) + operations["variables"]["input"]["name"] = name_with_batch_number + + self.__create_project_from_chunk(operations) + + def __get_chunked_gql_documents(self): + gql_documents = GraphQLDocumentCreator( + proxy_url=self.proxy_url, headers=self.headers, documents_path=self.documents_path).create() + + return [gql_documents[i:i + self.document_batch_size] for i in range(0, len(gql_documents), self.document_batch_size)] + + def __create_project_from_chunk(self, operations): + self.__report_before_request(operations) + + graphql_response = self.graphql_utils.call_graphql( + data={ + "query": operations["query"], + "variables": json.dumps(operations["variables"]), + "operationName": operations.get( + "operationName", "Datasaur API client - createProject" + ), + } + ) + + self.graphql_utils.process_graphql_response(graphql_response) + + def __report_before_request(self, operations): + print("\n" + "=" * 50 + "\n") + print("Project creation request sent.") + print(f"Project name: {operations['variables']['input']['name']}") + print( + f"Number of documents: {len(operations['variables']['input']['documents'])}") + print( + f"Document names: {', '.join(doc['document']['name'] for doc in operations['variables']['input']['documents'])}") + print("") + + def __get_name_with_batch_number(self, name: str, index: int): + return f"{name} - batch {index + 1}" if name else None diff --git a/create-project-async/src/graphql_document_creator.py b/create-project-async/src/graphql_document_creator.py new file mode 100644 index 0000000..bedadd1 --- /dev/null +++ b/create-project-async/src/graphql_document_creator.py @@ -0,0 +1,103 @@ +import glob +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import TypedDict + +from requests import post + + +class GraphQLDocument(TypedDict): + document: dict + extras: list[dict] | None + + +class GraphQLDocumentCreator: + MAX_WORKERS = 8 + + def __init__(self, proxy_url: str, headers: dict[str, str], documents_path: str): + self.proxy_url = proxy_url + self.headers = headers + self.documents_path = documents_path + + def create(self): + mapped_documents = self.__get_mapped_documents() + return self.__get_graphql_documents(mapped_documents) + + def __get_mapped_documents(self): + filepaths = list(glob.iglob(f"{self.documents_path}/*")) + sorted_filepaths = self.__sort_possible_extra_files_last(filepaths) + mapped_documents = self.__map_documents(sorted_filepaths) + return mapped_documents + + def __get_graphql_documents(self, mapped_documents: dict[str, dict]): + print(f"uploading files using {self.MAX_WORKERS} workers") + graphql_documents: list[GraphQLDocument] = [] + with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor: + futures = [ + executor.submit( + self.__upload_and_create_document, + key=key, + mapped_documents=mapped_documents, + ) + for key in mapped_documents.keys() + ] + + uploaded = 0 + for future in as_completed(futures): + gql_document = future.result() + uploaded += 1 + print(f"uploaded {uploaded}/{len(futures)} files", end="\r") + graphql_documents.append(gql_document) + + print() + + return graphql_documents + + def __upload_and_create_document(self, key: str, mapped_documents: dict[str, dict]): + upload_document_response = self.__upload_file(mapped_documents[key]["document"]) + document: GraphQLDocument = { + "document": { + "name": os.path.basename(mapped_documents[key]["document"]), + "objectKey": upload_document_response["objectKey"], + }, + "extras": None, + } + + if "extra" in mapped_documents[key]: + upload_extra_response = self.__upload_file( + filepath=mapped_documents[key]["extra"] + ) + document["extras"] = [ + { + "name": os.path.basename(mapped_documents[key]["extra"]), + "objectKey": upload_extra_response["objectKey"], + } + ] + return document + + def __upload_file(self, filepath: str): + with post( + url=self.proxy_url, + headers=self.headers, + files=[("file", open(filepath, "rb"))], + ) as response: + response.raise_for_status() + return response.json() + + def __sort_possible_extra_files_last(self, filepaths: list[str]): + # Sort file paths ending with .json, .txt, or .xml to be at the end + EXTRA_FILES_EXTENSIONS = (".json", ".txt", ".xml") + return sorted( + filepaths, + key=lambda x: (os.path.splitext(x)[-1] in EXTRA_FILES_EXTENSIONS, x), + ) + + def __map_documents(self, filepaths: list[str]): + mapped_documents: dict[str, dict] = {} + for filepath in filepaths: + filename = os.path.basename(filepath).split(".")[0] + if filename in mapped_documents: + mapped_documents[filename]["extra"] = filepath + else: + mapped_documents[filename] = {"document": filepath} + return mapped_documents diff --git a/create-project-async/src/graphql_utils.py b/create-project-async/src/graphql_utils.py new file mode 100644 index 0000000..3e1d408 --- /dev/null +++ b/create-project-async/src/graphql_utils.py @@ -0,0 +1,30 @@ +import json + +from requests import post + + +class GraphQLUtils: + def __init__(self, base_url, client_id, client_secret, headers): + self.base_url = base_url + self.graphql_url = f"{base_url}/graphql" + self.client_id = client_id + self.client_secret = client_secret + self.headers = headers + + def call_graphql(self, data): + return post(url=self.graphql_url, headers=self.headers, data=data) + + def process_graphql_response(self, response): + if "json" in response.headers["content-type"]: + json_response: dict = json.loads(response.text.encode("utf8")) + if "errors" in json_response: + print(json.dumps(json_response["errors"], indent=1)) + else: + job = json_response["data"]["result"]["job"] + print(json.dumps(job, indent=1)) + print("Check job status using the command below") + get_job_status_command = f"python api_client.py get_job_status --base_url {self.base_url} --client_id {self.client_id} --client_secret {self.client_secret} --job_id {job['id']}" + print(get_job_status_command) + else: + print(response.text.encode("utf8")) + print(response) diff --git a/create-project-async/src/helper.py b/create-project-async/src/helper.py index 2f716b8..815ca01 100644 --- a/create-project-async/src/helper.py +++ b/create-project-async/src/helper.py @@ -5,6 +5,7 @@ def get_access_token(base_url, client_id, client_secret): + print("Getting access token...") client = BackendApplicationClient(client_id=client_id) oauth = OAuth2Session(client=client) token = oauth.fetch_token( @@ -12,6 +13,7 @@ def get_access_token(base_url, client_id, client_secret): client_id=client_id, client_secret=client_secret, ) + print("Access token received.") return token["access_token"] diff --git a/create-project-async/src/project.py b/create-project-async/src/project.py index 8a19c67..77d4054 100644 --- a/create-project-async/src/project.py +++ b/create-project-async/src/project.py @@ -1,69 +1,43 @@ -import glob import json import os -from requests import post +from src.graphql_document_creator import GraphQLDocument, GraphQLDocumentCreator +from src.graphql_utils import GraphQLUtils from src.helper import get_access_token, get_operations class Project: - def __init__(self, base_url: str, id: str, secret: str): + def __init__(self, base_url: str, id: str, secret: str, documents_path: str): + if os.path.isfile(documents_path): + raise NotImplementedError( + "createProject with a list of documents is not yet implemented" + ) + self.base_url = base_url self.graphql_url = f"{base_url}/graphql" self.proxy_url = f"{base_url}/api/static/proxy/upload" self.client_id = id self.client_secret = secret - self.headers = None - - def create(self, team_id, operations_path, documents_path, name=None): - if os.path.isfile(documents_path): - raise NotImplementedError( - "createProject with a list of documents is not yet implemented" - ) + self.documents_path = documents_path access_token = get_access_token( self.base_url, self.client_id, self.client_secret ) - self.headers = self.__add_headers( - key="Authorization", value=f"Bearer {access_token}" - ) - operations = get_operations(operations_path) + self.headers: dict[str, str] = { + "Authorization": f"Bearer {access_token}", + } - operations["variables"]["input"]["documents"] = [] - operations["variables"]["input"]["teamId"] = team_id - - filepaths = list(glob.iglob(f"{documents_path}/*")) - sorted_filepaths = self.__sort_possible_extra_files_last(filepaths) - mapped_documents = self.__map_documents(sorted_filepaths) - - for key in mapped_documents: - upload_document_response = self.__upload_file( - filepath=mapped_documents[key]["document"] - ) - documents = { - "document": { - "name": os.path.basename(mapped_documents[key]["document"]), - "objectKey": upload_document_response["objectKey"], - } - } - - if "extra" in mapped_documents[key]: - upload_extra_response = self.__upload_file( - filepath=mapped_documents[key]["extra"] - ) - documents["extras"] = [ - { - "name": os.path.basename(mapped_documents[key]["extra"]), - "objectKey": upload_extra_response["objectKey"], - } - ] + def create(self, team_id: str, operations_path: str, name: str | None = None): + gql_documents = GraphQLDocumentCreator( + proxy_url=self.proxy_url, headers=self.headers, documents_path=self.documents_path).create() - operations["variables"]["input"]["documents"].append(documents) + operations = self._get_operations( + team_id, operations_path, gql_documents, name) - if name is not None: - operations["variables"]["input"]["name"] = name + gql = GraphQLUtils(base_url=self.base_url, headers=self.headers, + client_id=self.client_id, client_secret=self.client_secret) - graphql_response = self.__call_graphql( + graphql_response = gql.call_graphql( data={ "query": operations["query"], "variables": json.dumps(operations["variables"]), @@ -73,54 +47,14 @@ def create(self, team_id, operations_path, documents_path, name=None): } ) - self.__process_graphql_response(graphql_response) + gql.process_graphql_response(graphql_response) - def __upload_file(self, filepath): - with post( - url=self.proxy_url, - headers=self.headers, - files=[("file", open(filepath, "rb"))], - ) as response: - response.raise_for_status() - return response.json() - - def __call_graphql(self, data): - return post(url=self.graphql_url, headers=self.headers, data=data) - - def __process_graphql_response(self, response): - if "json" in response.headers["content-type"]: - json_response: dict = json.loads(response.text.encode("utf8")) - if "errors" in json_response: - print(json.dumps(json_response["errors"], indent=1)) - else: - job = json_response["data"]["result"]["job"] - print(json.dumps(job, indent=1)) - print("Check job status using command bellow") - get_job_status_command = f"python api_client.py get_job_status --base_url {self.base_url} --client_id {self.client_id} --client_secret {self.client_secret} --job_id {job['id']}" - print(get_job_status_command) - else: - print(response.text.encode("utf8")) - print(response) - - def __add_headers(self, key, value): - if self.headers is None: - self.headers = {key: value} - else: - self.headers[key] = value - - return self.headers + def _get_operations(self, team_id: str, operations_path: str, documents: list[GraphQLDocument], name: str | None): + operations = get_operations(operations_path) + operations["variables"]["input"]["teamId"] = team_id + if name is not None: + operations["variables"]["input"]["name"] = name - def __sort_possible_extra_files_last(self, filepaths): - # Sort file paths ending with .json or .txt to be at the end - filepaths.sort(key=lambda x: (x.endswith(".json") or x.endswith(".txt"), x)) - return filepaths + operations["variables"]["input"]["documents"] = documents - def __map_documents(self, filepaths): - mapped_documents = {} - for filepath in filepaths: - filename = os.path.basename(filepath).split(".")[0] - if filename in mapped_documents: - mapped_documents[filename]["extra"] = filepath - else: - mapped_documents[filename] = {"document": filepath} - return mapped_documents + return operations From 8e8d686a8ebb26f29bd350185b65e1a634ec4501 Mon Sep 17 00:00:00 2001 From: Ivan Marcellino Date: Sat, 6 Jul 2024 22:11:54 +0700 Subject: [PATCH 4/7] Fix: datasaur_schemas_to_coco failed to run on Windows (#33) - adjust zipfile reading process to always use `/` - use Python's built-in module for creating temporary folder - adjustment on ids: ensure uniqueness using array length and start at 1 --- .../src/datasaur_schemas_to_coco.py | 102 ++++++++++-------- 1 file changed, 57 insertions(+), 45 deletions(-) diff --git a/bounding-boxes/src/datasaur_schemas_to_coco.py b/bounding-boxes/src/datasaur_schemas_to_coco.py index 1f3d42e..397ed03 100644 --- a/bounding-boxes/src/datasaur_schemas_to_coco.py +++ b/bounding-boxes/src/datasaur_schemas_to_coco.py @@ -7,10 +7,18 @@ from zipfile import Path as ZipPath from zipfile import ZipFile import logging +import tempfile from common.logger import log as _log -from formats.coco import COCO, COCOAnnotation, COCOCategory, COCOImage +from formats.coco import ( + COCO, + COCOAnnotation, + COCOCategory, + COCOImage, + COCOLicense, + COCOInfo, +) def log(message, level=logging.DEBUG, **kwargs): @@ -50,6 +58,9 @@ def datasaur_schemas_to_coco( "year": 2024, } + coco_info = COCOInfo(**info) + coco_licenses = [COCOLicense(**l) for l in licenses] + schemas = [s for s in schema_objects] # assuming all DatasaurSchema are from the same project, @@ -59,7 +70,7 @@ def datasaur_schemas_to_coco( # images from datasaur schemas -- name, dimension info images: list[COCOImage] = [ coco_images_from_datasaur_schema(id, schema) - for id, schema in enumerate(schemas) + for id, schema in enumerate(schemas, start=1) ] # annotations from bboxLabels @@ -69,8 +80,8 @@ def datasaur_schemas_to_coco( return asdict( COCO( - info=info, - licenses=licenses, + info=coco_info, + licenses=coco_licenses, categories=categories, images=images, annotations=annotations, @@ -100,36 +111,33 @@ def main() -> None: logging.basicConfig(level=args.log_level, format="%(message)s") export_zip = os.path.abspath(args.zip_filepath) - temp_destination = os.path.abspath("./temp/") - os.makedirs(temp_destination, exist_ok=True) - log("creating temp directory", directory=temp_destination) - - outfile = os.path.abspath(args.outfile) - outdir = os.path.dirname(outfile) - os.makedirs(outdir, exist_ok=True) - - extracted_files: list[str] = unzip_export_result( - export_zip=export_zip, dest=temp_destination - ) + with tempfile.TemporaryDirectory() as temp_destination: + log("using temp directory", directory=temp_destination) + outfile = os.path.abspath(args.outfile) + outdir = os.path.dirname(outfile) + os.makedirs(outdir, exist_ok=True) + + extracted_files: list[str] = unzip_export_result( + export_zip=export_zip, dest=temp_destination + ) - schemas = [load_datasaur_schema_file(f) for f in extracted_files] + schemas = [load_datasaur_schema_file(f) for f in extracted_files] - log("reading licenses and info file", filepath=args.license_and_info_json) - license_and_info = json.load(open(os.path.abspath(args.license_and_info_json))) + log("reading licenses and info file", filepath=args.license_and_info_json) + license_and_info = json.load(open(os.path.abspath(args.license_and_info_json))) - log("converting datasaur schemas to COCO format", count=len(schemas)) - coco = datasaur_schemas_to_coco( - schemas, - licenses=license_and_info.get("licenses", None), - info=license_and_info.get("info", None), - ) + log("converting datasaur schemas to COCO format", count=len(schemas)) + coco = datasaur_schemas_to_coco( + schemas, + licenses=license_and_info.get("licenses", None), + info=license_and_info.get("info", None), + ) - log("writing COCO JSON file", outfile=outfile) - with open(outfile, "w") as wf: - json.dump(coco, wf, indent=2) + log("writing COCO JSON file", outfile=outfile) + with open(outfile, "w") as wf: + json.dump(coco, wf, indent=2) - log("cleaning up temp directory", directory=temp_destination) - rmtree(temp_destination) + log("cleaning up temp directory", directory=temp_destination) def coco_annots_from_datasaur_schemas( @@ -138,9 +146,7 @@ def coco_annots_from_datasaur_schemas( name_to_id: Dict[str, int] = {x.name: x.id for x in categories} annots: list[COCOAnnotation] = [] - for image_id, schema in enumerate(schemas): - annot_id = 1 - + for image_id, schema in enumerate(schemas, start=1): if ( schema["data"]["bboxLabelSets"] is None or len(schema["data"]["bboxLabels"]) < 1 @@ -164,13 +170,14 @@ def coco_annots_from_datasaur_schemas( bbox_label["bboxLabelClassId"], None ) - for key, value in answers.items(): - question_label = questions[key]["label"] - attributes[question_label] = value + if questions: + for key, value in answers.items(): + question_label = questions[key]["label"] + attributes[question_label] = value annots.append( COCOAnnotation( - id=annot_id, + id=len(annots) + 1, image_id=image_id, category_id=name_to_id[bbox_label["bboxLabelClassName"]], segmentation=shapes_to_segmentation(bbox_label["shapes"]), @@ -203,9 +210,13 @@ def coco_categories_from_datasaur_schema(schema: dict) -> list[COCOCategory]: def coco_images_from_datasaur_schema(id: int, schema: dict) -> COCOImage: width, height = 0, 0 - if schema["data"]["pages"] is not None and len(schema["data"]["pages"]) >= 1: - width = schema["data"]["pages"][0]["pageWidth"] - height = schema["data"]["pages"][0]["pageHeight"] + try: + if schema["data"]["pages"] is not None and len(schema["data"]["pages"]) >= 1: + width = schema["data"]["pages"][0]["pageWidth"] + height = schema["data"]["pages"][0]["pageHeight"] + except KeyError: + # some older Bounding Box projects may not have pageWidth / pageHeight populated + pass return COCOImage( id=id, @@ -243,18 +254,19 @@ def unzip_export_result(export_zip: str, dest: str) -> list[str]: retval: list[str] = [] log("unzipping export result to temp directory", export_zip=export_zip) with ZipFile(export_zip, "r") as zf: - project_dir: str | None = None + project_dirs: list[str] = [] + for zippath in ZipPath(zf).iterdir(): if zippath.is_dir(): - project_dir = zippath.name - break - if not (project_dir): + project_dirs.append(zippath.name) + + if len(project_dirs) == 0: log("no project dir found in export result", level=logging.ERROR) raise Exception("no project dir found") - log("project_dir", project_dir=project_dir) + prefixes = [f"{project_dir}/REVIEW" for project_dir in project_dirs] for zip_content in zf.infolist(): - if not zip_content.filename.startswith(os.path.join(project_dir, "REVIEW")): + if not any(zip_content.filename.startswith(prefix) for prefix in prefixes): continue if zip_content.is_dir(): From 85bde4c1b599ef4fbb08fbe2d674081059be8032 Mon Sep 17 00:00:00 2001 From: Billy Editiano <102159897+billy-editiano@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:17:18 +0700 Subject: [PATCH 5/7] Add Create Project via EOS (#34) * Add Create Project via EOS * address reviews suggestions --- create_project_via_eos.json | 88 +++++++++++++++++++++++++++++++++++++ create_project_via_eos.py | 23 ++++++++++ 2 files changed, 111 insertions(+) create mode 100644 create_project_via_eos.json create mode 100644 create_project_via_eos.py diff --git a/create_project_via_eos.json b/create_project_via_eos.json new file mode 100644 index 0000000..db31150 --- /dev/null +++ b/create_project_via_eos.json @@ -0,0 +1,88 @@ +{ + "operationName": "CreateProjectMutation", + "variables": { + "input": { + "teamId": "", + "externalObjectStorageId": "", + "name": "Demo via API", + "documents": [ + { + "document": { + "name": "file-1.mp3", + "objectKey": "path/to/file/in/eos/file-1.mp3" + }, + "extras": [ + { + "name": "file-1.json", + "objectKey": "path/to/file/in/eos/file-1.json" + } + ] + } + ], + "documentAssignments": [ + { + "email": "demo@example.com", + "documents": [ + { + "fileName": "file-1.mp3", + "part": 0 + } + ], + "role": "LABELER_AND_REVIEWER" + }, + { + "email": "demo.labeler-1@example.com", + "documents": [ + { + "fileName": "file-1.mp3", + "part": 0 + } + ], + "role": "LABELER" + } + ], + "kinds": [ + "TOKEN_BASED" + ], + "purpose": "LABELING", + "creationSettings": { + "anonymizationConfig": null, + "customHeaderColumns": [ + { + "name": "Column 1", + "displayed": true, + "labelerRestricted": false + } + ], + "enableTabularMarkdownParsing": false, + "fileTransformerId": null, + "firstRowAsHeader": false, + "sentenceSeparator": "\n", + "splitDocumentConfig": null, + "tokenizer": "WINK", + "transcriptConfig": { + "method": "TRANSCRIPTION" + }, + "viewer": { + "mode": "TOKEN" + } + }, + "tokenLabelSets": [], + "rowQuestions": null, + "documentQuestions": null, + "bboxLabelSets": null, + "kindsDocumentSettings": { + "tokenBasedSettings": { + "allTokensMustBeLabeled": false, + "allowArcDrawing": false, + "allowCharacterBasedLabeling": false, + "allowMultiLabels": true, + "editSentenceTokenizer": "WINK", + "textLabelMaxTokenLength": 999999, + "autoScrollWhenLabeling": true + } + } + } + }, + "query": "mutation CreateProjectMutation($input: LaunchProjectInput!) { result: createProject(input: $input) { name job { id status progress resultId errors { id stack args } } } }" +} \ No newline at end of file diff --git a/create_project_via_eos.py b/create_project_via_eos.py new file mode 100644 index 0000000..abd97c4 --- /dev/null +++ b/create_project_via_eos.py @@ -0,0 +1,23 @@ +import fire +import json +import os + +from toolbox.get_access_token import get_access_token +from toolbox.get_operations import get_operations +from toolbox.post_request import post_request + +def create_project_via_eos(base_url, client_id, client_secret): + url = base_url + "/graphql" + access_token = get_access_token(base_url, client_id, client_secret) + operations = get_operations("create_project_via_eos.json") + + response = post_request(url, access_token, operations) + if "json" in response.headers["content-type"]: + json_response = json.loads(response.text.encode("utf8")) + return json_response + else: + return response.text + +if __name__ == "__main__": + os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" + fire.Fire(create_project_via_eos) From 45ae505ae04504852b7bd81a1423e9627dae0ee6 Mon Sep 17 00:00:00 2001 From: Ivan Marcellino Date: Mon, 23 Sep 2024 08:11:44 +0700 Subject: [PATCH 6/7] Enhance bbox_label_class attributes' id generation (#35) * skip fetching id from custom_labelset, use array indices as unique id * intentionally set custom-label-set's id to be duplicated * also add duplicate internalId --- bounding-boxes/samples/custom-label-set.json | 4 +++- bounding-boxes/src/coco_to_datasaur_schemas.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bounding-boxes/samples/custom-label-set.json b/bounding-boxes/samples/custom-label-set.json index f3e7574..47e9428 100644 --- a/bounding-boxes/samples/custom-label-set.json +++ b/bounding-boxes/samples/custom-label-set.json @@ -6,6 +6,7 @@ "questions": [ { "id": 0, + "internalId": 123, "label": "bio_tag", "required": false, "type": "DROPDOWN", @@ -25,7 +26,8 @@ } }, { - "id": 1, + "id": 0, + "internalId": 123, "label": "label-2", "type": "TEXT", "required": false, diff --git a/bounding-boxes/src/coco_to_datasaur_schemas.py b/bounding-boxes/src/coco_to_datasaur_schemas.py index 431e35b..db50604 100644 --- a/bounding-boxes/src/coco_to_datasaur_schemas.py +++ b/bounding-boxes/src/coco_to_datasaur_schemas.py @@ -169,7 +169,7 @@ def bbox_label_classes_from_coco( questions = [ DSBBoxLabelClassQuestions( - id=q.get("id", index), + id=index, label=q.get("label", f"Question {index}"), config=QuestionConfig( multiline=q.get("config", {}).get("multiline", None), From 7e86b65ece93b745079ec5e94afa311e2945a355 Mon Sep 17 00:00:00 2001 From: Ivan Marcellino Date: Tue, 24 Feb 2026 17:58:16 +0700 Subject: [PATCH 7/7] Add export-eos.json for project export using External Object Storage (#37) also: format export.py using Black formatter --- export-eos.json | 17 +++++++++++++ export.py | 65 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 61 insertions(+), 21 deletions(-) create mode 100644 export-eos.json diff --git a/export-eos.json b/export-eos.json new file mode 100644 index 0000000..8eecb36 --- /dev/null +++ b/export-eos.json @@ -0,0 +1,17 @@ +{ + "operationName": "ExportTextProjectQuery", + "variables": { + "input": { + "projectIds": [""], + "role": "REVIEWER", + "format": "DATASAUR_SCHEMA", + "fileName": "", + "method": "EXTERNAL_OBJECT_STORAGE", + "externalObjectStorageParameter": { + "externalObjectStorageId": "", + "prefix": "" + } + } + }, + "query": "query ExportTextProjectQuery($input: ExportTextProjectInput!) {\n result: exportTextProject(input: $input) {\n ...ExportRequestRedirectResultFragment\n __typename\n }\n}\n\nfragment ExportRequestRedirectResultFragment on ExportRequestResult {\n exportId\n fileUrl\n queued\n redirect\n key\n __typename\n}\n" +} diff --git a/export.py b/export.py index 605a263..62df076 100644 --- a/export.py +++ b/export.py @@ -11,21 +11,30 @@ POOLING_INVERVAL = 0.5 # 0.5s -def export_project(base_url, client_id, client_secret, project_id, export_file_name, export_format, output_dir): +def export_project( + base_url, + client_id, + client_secret, + project_id, + export_file_name, + export_format, + output_dir, + operation_path="export.json", +): url = base_url + "/graphql" access_token = get_access_token(base_url, client_id, client_secret) - operations = get_operations('export.json') + operations = get_operations(operation_path) operations["variables"]["input"]["fileName"] = export_file_name operations["variables"]["input"]["projectIds"] = [project_id] operations["variables"]["input"]["format"] = export_format response = post_request(url, access_token, operations) - if 'json' in response.headers['content-type']: - json_response = json.loads(response.text.encode('utf8')) + if "json" in response.headers["content-type"]: + json_response = json.loads(response.text.encode("utf8")) print(json.dumps(json_response, indent=1)) - if len(json_response["data"]["result"]["fileUrl"]) > 0: + if json_response["data"]["result"]["fileUrl"]: export_id = json_response["data"]["result"]["exportId"] poll_export_delivery_status(url, access_token, export_id) @@ -34,28 +43,36 @@ def export_project(base_url, client_id, client_secret, project_id, export_file_n os.makedirs(output_dir, exist_ok=True) file_response_url = urlparse(file_url) file_name = os.path.basename(file_response_url.path) - output_file = output_dir + '/' + file_name - open(output_file, 'wb').write(file_response.content) + output_file = output_dir + "/" + file_name + open(output_file, "wb").write(file_response.content) return "Success downloading the file. Output file:" + output_file + else: + export_id = json_response["data"]["result"]["exportId"] + poll_export_delivery_status(url, access_token, export_id) + return ( + "Success exporting the project. Check your storage bucket for the file." + ) else: return response def poll_export_delivery_status(url, access_token, export_id): - operations = get_operations('get_export_delivery_status.json') + operations = get_operations("get_export_delivery_status.json") operations["variables"]["exportId"] = export_id while True: time.sleep(POOLING_INVERVAL) response = post_request(url, access_token, operations) - if 'json' in response.headers['content-type']: - json_response = json.loads(response.text.encode('utf8')) - delivery_status = json_response["data"]["exportDeliveryStatus"]["deliveryStatus"] - if (delivery_status == "QUEUED"): + if "json" in response.headers["content-type"]: + json_response = json.loads(response.text.encode("utf8")) + delivery_status = json_response["data"]["exportDeliveryStatus"][ + "deliveryStatus" + ] + if delivery_status == "QUEUED": print("Waiting for exported file to be ready...") - elif (delivery_status == "DELIVERED"): + elif delivery_status == "DELIVERED": print("Exported file is ready") break - elif (delivery_status == "FAILED"): + elif delivery_status == "FAILED": print("Failed to export file") break else: @@ -66,21 +83,27 @@ def poll_export_delivery_status(url, access_token, export_id): def get_access_token(base_url, client_id, client_secret): client = BackendApplicationClient(client_id=client_id) oauth = OAuth2Session(client=client) - token = oauth.fetch_token(token_url=base_url + '/api/oauth/token', client_id=client_id, - client_secret=client_secret) - return token['access_token'] + token = oauth.fetch_token( + token_url=base_url + "/api/oauth/token", + client_id=client_id, + client_secret=client_secret, + ) + return token["access_token"] def get_operations(file_name): - with open(file_name, 'r') as file: + with open(file_name, "r") as file: return json.loads(file.read()) def post_request(url, access_token, operations): - headers = {'Authorization': 'Bearer ' + access_token, 'Content-Type': 'application/json'} + headers = { + "Authorization": "Bearer " + access_token, + "Content-Type": "application/json", + } return requests.request("POST", url, headers=headers, data=json.dumps(operations)) -if __name__ == '__main__': - os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' +if __name__ == "__main__": + os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" fire.Fire(export_project)