From b6ff6e5defe2c0743010dcfa8a50db31fe7fb15c Mon Sep 17 00:00:00 2001 From: Alexander VT Date: Mon, 1 Nov 2021 16:47:01 -0500 Subject: [PATCH 1/3] feat(profiles): simplify configuration and use standard .ini library, allow using --auth profile and getting api key from configuration --- gen3/auth.py | 9 ++++++-- gen3/cli/configure.py | 5 ++--- gen3/configure.py | 52 +++++++++++++++---------------------------- gen3/utils.py | 22 ++++++++++++++++++ 4 files changed, 49 insertions(+), 39 deletions(-) diff --git a/gen3/auth.py b/gen3/auth.py index bc8873043..9adab4c66 100755 --- a/gen3/auth.py +++ b/gen3/auth.py @@ -10,7 +10,7 @@ from urllib.parse import urlparse import backoff -from gen3.utils import DEFAULT_BACKOFF_SETTINGS, raise_for_status +from gen3.utils import DEFAULT_BACKOFF_SETTINGS, raise_for_status, get_cfg_from_profile class Gen3AuthError(Exception): @@ -164,7 +164,12 @@ def __init__(self, endpoint=None, refresh_file=None, refresh_token=None, idp=Non if not refresh_file and not refresh_token and not idp: refresh_file = os.getenv("GEN3_API_KEY", "credentials") - if refresh_file and not idp: + # prefer to check for a configure profile with the provided name first + if get_cfg_from_profile(profile=refresh_file, cfg="api_key_filepath"): + refresh_file = get_cfg_from_profile( + profile=refresh_file, cfg="api_key_filepath" + ) + elif refresh_file and not idp: idp_prefix = "idp://wts/" access_token_prefix = "accesstoken:///" if refresh_file[0 : len(idp_prefix)] == idp_prefix: diff --git a/gen3/cli/configure.py b/gen3/cli/configure.py index c8e39a5c8..1c664479e 100644 --- a/gen3/cli/configure.py +++ b/gen3/cli/configure.py @@ -15,9 +15,8 @@ def configure(profile, cred): logging.info(f"Configuring profile [ {profile} ] with credentials at {cred}") try: - profile_title, new_lines = config_tool.get_profile_from_creds(profile, cred) - lines = config_tool.get_current_config_lines() - config_tool.update_config_lines(lines, profile_title, new_lines) + cfg = config_tool.get_profile_from_creds(cred) + config_tool.update_config_lines(profile, cfg) except Exception as e: logging.warning(str(e)) raise e diff --git a/gen3/configure.py b/gen3/configure.py index cd13978a1..1323776a0 100644 --- a/gen3/configure.py +++ b/gen3/configure.py @@ -19,58 +19,42 @@ """ import json -import logging -from os.path import expanduser +from cdislogging import get_logger +import configparser +from os.path import expanduser, abspath from pathlib import Path from collections import OrderedDict -import gen3.auth as auth_tool +import gen3.auth as auth_tool +from gen3.utils import CONFIG_FILE_PATH -CONFIG_FILE_PATH = expanduser("~/.gen3/config") +logging = get_logger("__name__") -def get_profile_from_creds(profile, cred): +def get_profile_from_creds(cred): with open(expanduser(cred)) as f: creds_from_json = json.load(f) credentials = OrderedDict() credentials["key_id"] = creds_from_json["key_id"] credentials["api_key"] = creds_from_json["api_key"] + credentials["api_key_filepath"] = abspath(f.name) credentials["api_endpoint"] = auth_tool.endpoint_from_token( credentials["api_key"] ) - credentials["access_key"] = auth_tool.get_access_token_with_key(credentials) credentials["use_shepherd"] = "" credentials["min_shepherd_version"] = "" - profile_line = "[" + profile + "]\n" - new_lines = [key + "=" + value + "\n" for key, value in credentials.items()] - new_lines.append("\n") # Adds an empty line between two profiles. - return profile_line, new_lines - -def get_current_config_lines(): - """ Read lines from the config file if exists in ~/.gen3 folder, else create new config file """ - try: - with open(CONFIG_FILE_PATH) as configFile: - logging.info(f"Reading existing config file at {CONFIG_FILE_PATH}") - return configFile.readlines() - except FileNotFoundError: - Path(CONFIG_FILE_PATH).touch() - logging.info(f"Config file doesn't exist at {CONFIG_FILE_PATH}, creating one") - return [] + return credentials -def update_config_lines(lines, profile_title, new_lines): - """ Update config file contents with the new profile values """ +def update_config_lines(profile, creds): + """ + Update config file contents with the new profile values + """ + config = configparser.ConfigParser() + config.read(CONFIG_FILE_PATH) - if profile_title in lines: - profile_line_index = lines.index(profile_title) - next_profile_index = len(lines) - for i in range(profile_line_index, len(lines)): - if lines[i][0] == "[": - next_profile_index = i - break - del lines[profile_line_index:next_profile_index] + config[profile] = creds - with open(CONFIG_FILE_PATH, "a+") as configFile: - configFile.write(profile_title) - configFile.writelines(new_lines) + with open(CONFIG_FILE_PATH, "w") as file: + config.write(file) diff --git a/gen3/utils.py b/gen3/utils.py index f92b448ff..cf6cbb4bd 100644 --- a/gen3/utils.py +++ b/gen3/utils.py @@ -2,12 +2,15 @@ import sys import re import requests +import configparser +from os.path import expanduser from urllib.parse import urlunsplit from urllib.parse import urlencode from urllib.parse import urlsplit from urllib.parse import parse_qs +CONFIG_FILE_PATH = expanduser("~/.gen3/config") UUID_FORMAT = ( r"^.*[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$" @@ -166,6 +169,25 @@ def get_urls(raw_urls_string): ] +def get_cfg_from_profile(profile, cfg): + """ + Return value of `cfg` in configure profile `profile` or None if either don't exist + Args: + profile (str): name of profile previously setup using `gen3 configure` + cfg (str): name of configuration variable within the profile + + Returns: + str: Value of `cfg` in provided configure profil + """ + config = configparser.ConfigParser() + config.read(CONFIG_FILE_PATH) + + try: + return config[profile][cfg] + except Exception as exc: + return None + + # Default settings to control usage of backoff library. DEFAULT_BACKOFF_SETTINGS = { "on_backoff": log_backoff_retry, From 89e5c2075057dbfbde1fda2fbf3d8e2928f2c706 Mon Sep 17 00:00:00 2001 From: Avantol13 Date: Mon, 1 Nov 2021 21:50:17 +0000 Subject: [PATCH 2/3] Apply automatic documentation changes --- docs/_build/doctrees/environment.pickle | Bin 225740 -> 226046 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 93370 -> 93366 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 34277 -> 34277 bytes docs/_build/html/_modules/gen3/auth.html | 9 +++++++-- docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- 7 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index dd3923a5d50456bf128824f61f48c7fcf5fbe488..382af6d6e5b8588cad93ee55c51ed5c5d1689576 100644 GIT binary patch delta 4633 zcmZ`7dsNhA*6%mN3?eGT`#ms&0>TKIiZC$XD|rkL$E>u(5eA%}zzp-i+(OwrQRq)l z@90k+G9Z|mRM=Fsd+MAzyU{soxml;S*Or|&TeH&IZ6Dd{-tYU(47R!aaliY%_q+G| zefJyQ_|kh_;xWrUsMKgwi&b*}l^)%wZlDP7CCXmS(eC2ywW!wpbhvi?ubN6mWFFwN+5I zo9$kr`U|&vkee~%uG06n#IBAT8sv`$sDy!Q{ILL)Sp6q{C_okV1@Zj>sx;>(-V>k< z^UC<)0A8~Bj`HQHt12LjaTP7?^~!eunsUU!Bt zSF2svWNux&kQPg8+vz^QGXI%~RzeaD!3}i+>4$htypASH)gt^J{I!!*Y35WM<`=U< zOR`b0X;h>S4r(-H3|>i(hS}*W<^(idPD`@`%F`X3umEmqV&Gj(lTyj24)crH1oa3V0(w7H-T_LTctZ zn9cMQFqmnD%L~+?&FX+Cv^k%(19oJq;cLAb_AS^99rM)?n!O!XqfURegN(qOoWpPi zJ$_oPfv<9Gtmix@n=yk`jZdT)?KPi8G1_N7pJKGf{2GeUvhnLFMk~Z`r5NoiZ=)FP zDZh(iV-yo8MqA1EQ;fEeKZTe}kUOpQ4tG$8JA)teexl7GaI+vCa&upUw(3?aLK1Z5 z8DT?>2L72h1^4DB!H0`Jgk{S$@c!bLcGDCErMcB>9me>|z$(Bt&-Ojw z&03PnVUppnec3l~xvRzT$AxYU7n*E^ zob3kf24JjqkTKTrWU|k~nOz<*bSp_Z_A4Lzvw2WptP5JH^ zpC-+FT1L~KrsyuFDS8;Esj+AuNtN&(@l){ex;O>YcVpK{*-F}$Zq}7TN; zWRA3d4`dx#jS99SdV~{4su12}xL%}qkElayuOp*S)nBe;_w7|BmC93-WDd6s`@HODTlFXUCR^bBnn!zn6W-YS7Ea$Ep!# z50xQoA2K5R!%!{4L{FJym;!l2d19`P^VN40COj*o!0GW`rIFZ;aF;UUS@x@P*?m5dQo4S_Qk)${=+tAHp6A@*OzFtfb6vW7)>l zzKSqzHF6{2n^7&=Cyx2-@20Aucdx57`c>iSu`# zMuay{v>bD9GS&WZb9gIaw!>r_nyp0IQL8ug!KnO?U^E^ zde1CVut&-ul#XXY_2a?5?hb%_d>Ja}#&rmHju#?)eq4|6hw%dJLhe&Zm~a(?%?$2E zIO09^)J%?>TI%&aBPY?>D*AoaH#XHZ+N~{hHoKKRGQ5)$BT>86Dx2NhXtt}IR#p8r zbNzNzqt&i5sp_q*jg}^t-HdXkuG(R7Zc{n8F-E(k!Q7x~HMN*kjTXDZ8KUz2SsHH} zE5qlc2xHmJj%{@`foxTMV@y*~n#pFV+iu>;cs7%BTUus82;6@@Hda-1duL+mk0c_C zW%Ub(xAP~y`WzPnuGhn1_PLo}+f<5*d;|wBCCdI>aT%Vulp?!gP{4&tnet#_NQ4`g z)beCvSP1D~rpxn)p&Y8d)HGBQgONPu7G~dtkN#5n=C60zo8A2PAv0ze?j{BY6$bl- zZoURD$BJggcHv|5RuP>-zwkquknv>_n^8g%6l_T5@P!)p*E}UorHZSmK+pr z62Ge<8g@Gi@nV~S#4FrvG>TROnJ=d^O02FTY6;s?Ly{$vDDlT?G8gd+wIm6btF?Ze zx2i~jirNV=H2*UCvrpS#ZZV}b9)mc_q(cy3VoefM|_MM z29hXgMd7U+InyEKCL~dcO|a!GQp9v5dNv(Slhdzy$InGrWuHVmTS?+D+CxURfj%Qq zNm=w(l2{4T)%dZ2Y7&pSVe9GsCl{_KvNSfZgcP`FQgM90xUrqYC(=13uw=^d7qIr$ zIo2;8Kn=+#5{NrmPL;(AP7)(A?||>>Bhb_2PwnJ41#Hv^anN}{}Q zB>BbFM-oz~SPE+J{hrp=+s!7Y+0EWH;%Cp0dBh_={H)(W_yl9T`YeWK#wxfbrlAV) zk`!E0y|_z{SmMR{c*Kof+`4DdEaT(u82%Tu)DwFKIg3P+}_O5^(7#^@WTSQ(|#eZ_&~0CVyATH7r!p@#7HE zq3z*FxcFfBq_K^2ZDQj-e+hmET^iZ2DK~jnp fJKJQD;zXNF5lw@*gy1Ip(6#ixJx;gFwN?9H&AuO? delta 4449 zcmZ`-c~nzp7WYd+f}jEwWM5(e0zwR0WfcgD$_@sLOIrw-BG0f$6417&Q%73#0m6Oq z1lfs8D>~TNGBc+=(>Z6P=Zsy)X-~(~#hz(rw5_e}Oxu~`ROf!*3&HVp{`r0P-tXS~ z-R0$e@8$023*Y$bLXU%r<68?adbC9zP&`p*F$?@6TVtcex}x4}Gwn3lOakw*$8K)k zC-4fcvDzx|o^=g7&3o$_YLUE8iCLjC+eKxQ-L^~M<*imUak-o`^s?Hk>+7u6y2b|K zn7x`;wV0coBi=~mR58&<8LN$Z;S3CCFE|FcDlD{T$IT}>z%gb;0riUx{m;1LrQ;5+Mu z)mzeaK$A9#j6jc8tD11>c1P<1c(=0MR%fwNWj+piq66VQZ7Qgv8|25h0eCHX6)0l@ z;JfI>$d5xrjFpT*Z_Jl)bGZgam)YT5yap0iJq(uEdI*Tsz+3T3cq1hU?#8G<9k&^y z+`lXq2ICsxFSx6VZ-a?Ms?`du2^#n+O9MxjKMvmLWkJGj$VIOM30APiX+WQ7g|=lH z_@qP&UnDlM#&RYx^O(DXur4tO3_|%XOXh=3R`21Rq$vNV!1Kr$GSuU?-luhZ@NF|^_QX0um(of zq+l_^;b4*xVsf^sf$31P08XQ@oAF6c1ORSs4O zRs@#+xHEpWUWP&X!S3W+@aoEN$V@A9b#XUrPo{}3h8?Lh*vSJ~^_Zp-yBP?RX-4qY zM}s!~GHgw0z|N{+Z-x<)Q?>9-#tiH!X+SR#V8}GWBc)pSb><9QUN4pEtE?9IVuR$l zJ$nXjrmulJ>r?Rw2!g8yBV5VQLVnIvI*sswlF?St)M=xBK6Td20f=AS>YQ8`OkjT& zb$%AAv$f9Gb2WraIKSK=(&f;mKDvbDpCq78TLWJfoP~R7TIY0O1VO?N;`0OGz%#+H ze50Q;y10RzT$Au<_ z3k|hE!E!;nad4EaTTh+f!|^j6{67sueKT`Ntz5}3-yFt%E;txiRPJfw;^;#$w?`wL z<6Hy9rAgzS+oR#naWsoLjy{ZY+{6+WOO9;a6>|C((eI;ybU*x>aRS@EEZnAiJ;Yc&Kti|^Aw2XL@ z0cX-Pu?jNo+}?ghzK*t~gLS2YrP0B9(!pBN!8+2x8q&e~(ZQb)C z5b+wnNy&bOg;2~VLc=z1sOnVWMs4Q?#0#BSh+lW+N=jSd$*yHk)#Z;udsjB%%Ux>_ zZ*}D$>bqTPweU*MGPvID!C+a_pFuF4zr^VR(UUQolHyf9He4A zmA#84oue#^kzSg`rCvP_kI#EIASOV%l0I+T2|PYJu-0s=Y_!{&>^4^xZ0{u9gSWsB z4X#55Vz`iwxJ^hy6oeEq3~vhAh>MQru^W)PsDhq}U}!xahtjd*IZ8JBr?Fp&;!`h@ zMN<015Z{-J3Qc`kh*N#Vi1!$sB~rRe%s{*%7E0!C!@K=~&|kL{5_`PCbb=aLa3B}V|hI;Te0kqP+cXaQpO*yD&J4DXJW zAzeCdMtpfZ2l3zIo0M#D<$>Cf0=`dqxE?GfETlp|W8U^$S1FjZ1f~9PYeI+iVGfu5 z7OJ{n%jQDktSVe^brFQJDEc72?PXYsfgf|3V6)&$+dTMdu8N-RF{(?0sYq43lwC^4#nXbQ~0u z$*7PqnSr=(G9B@y$t=Y0C)2P8`t#u!ZZSg(!y||T&a>y|%Vff8r}I||BBuSBEOcI* z5&~p(Pr&WVOPu4cM@B`x|E$3KZNyhn1AQZ{tgJCvER~f4_njBsSsxU92A}>tQu((c zLY@^M?xuP^l^4lLQ6T7VXcWPuC>|l&INE;zPi9Yp4gi3&gQvvP|0lMG4W6A@P;XM2*YGn~7G^UIOoQ2BAc4 zB${p16CX&U%cI{ex|(LQ$!4x>HrF&Z?-ZEJ`?q(RE#?QeS>lp4PCb`RF!*$bN^wk; zkSILvBO`GM^d%%r3MD`+EhYg<$wn+LW|8hFCn2FMu7~F0qVp8XYDHD?SUCwphv&;l zI4)PpNtATZ?~6!~L^YJnQg4=$5b3NyNHPWCOU$%s3z08nvzm{D>!v6fHz@M0BqWlK zXnrw0Gaonv~Dl? zO`3uZZ87Ra40u?a?3LroX_4z<wE^@+i11E|uK;Zi<9S)W8%84P`D^QQcm8J<3K ziSQ9h`f`MmylHn(Fngciedt+b^kY}&vN$PnSIJUxa=P;>sj?7<_;r>{D=iT!&Rm1h zI7DBA%%9x_a>_O3xHtv2(IK{FGXq>A#MiQAN;G~KHTpgyDlzk0qGUF>8%jA9q$FW&iwg9kt{gPPG5vjxZ08F0VQS^0Vb9b4 zaDA`e^x|H7(}*9# z?Bfcr(iP`|$hfF(ma8L}E_ox$pgUf+>#K9en6+W9(BZH&GInxbiP?0%7!@~ gB#t%7{l)GkxpE10Vdp(-v35H8hlfqD+pBf|1?!mUR{#J2 diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree index 0d8f735805583a736a3caab861e74d2c16716818..6a1782f326d7d535cabda6b8983e0780d6e6327c 100644 GIT binary patch delta 137 zcmdmWlXcro)`lsJm5ioas~A(28Q*Mwpv-uikL6uQk234b8PJX)jlnnORDH&ot5EbY~Z!%=$um%ACoh76I delta 141 zcmdmXlXcfk)`lsJm5i2Ks~A(28Q*Sypv-uik>A+F)WXETQqRoX(9qo6Wcx`~#wkpU z7q`c0Fz&~c)z)HUVrR6UK0%dHTNX{LUQT|x`jiaz*eMxeJrE@rrmr((SGZV|r=QRvj(8MO6*Ra{_rK`Y-E?!~egeJ84 Igpq780J{$ung9R* delta 92 zcmaFb&GfXJX~RJcK4TM83ll>FJwrny6C<Source code for gen3.auth
 from urllib.parse import urlparse
 import backoff
 
-from gen3.utils import DEFAULT_BACKOFF_SETTINGS, raise_for_status
+from gen3.utils import DEFAULT_BACKOFF_SETTINGS, raise_for_status, get_cfg_from_profile
 
 
 class Gen3AuthError(Exception):
@@ -197,7 +197,12 @@ 
Source code for gen3.auth
         if not refresh_file and not refresh_token and not idp:
             refresh_file = os.getenv("GEN3_API_KEY", "credentials")
 
-        if refresh_file and not idp:
+        # prefer to check for a configure profile with the provided name first
+        if get_cfg_from_profile(profile=refresh_file, cfg="api_key_filepath"):
+            refresh_file = get_cfg_from_profile(
+                profile=refresh_file, cfg="api_key_filepath"
+            )
+        elif refresh_file and not idp:
             idp_prefix = "idp://wts/"
             access_token_prefix = "accesstoken:///"
             if refresh_file[0 : len(idp_prefix)] == idp_prefix:
diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js
index 3c3055394..b1e9f7187 100644
--- a/docs/_build/html/searchindex.js
+++ b/docs/_build/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0934136":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"112426":10,"11e9":9,"1394515":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1633103143":[],"1633373719":[],"1633449426":[],"1633449427":[],"1633454066":[],"1633454067":[],"1633458409":9,"1633458410":10,"1731167":[],"197634":[],"2280114":[],"2343018":[],"255e396f":9,"2889535":[],"333":5,"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5859108":[],"5964222":[],"622057":[],"641011":[],"6542356":[],"6572418":[],"6711774":9,"6f90":7,"7461076":[],"7649791":[],"772628":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9781935":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],use_agg_md:5,used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}})
\ No newline at end of file
+Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"020972":9,"0339756":[],"0420947":[],"0934136":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"112426":[],"11e9":9,"1394515":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1633103143":[],"1633373719":[],"1633449426":[],"1633449427":[],"1633454066":[],"1633454067":[],"1633458409":[],"1633458410":[],"1635803414":[9,10],"1731167":[],"197634":[],"2280114":[],"2343018":[],"255e396f":9,"2889535":[],"333":5,"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"470649":10,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5859108":[],"5964222":[],"622057":[],"641011":[],"6542356":[],"6572418":[],"6711774":[],"6f90":7,"7461076":[],"7649791":[],"772628":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9781935":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],use_agg_md:5,used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}})
\ No newline at end of file
diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html
index 988bf898f..b2a9c1eda 100644
--- a/docs/_build/html/tools/indexing.html
+++ b/docs/_build/html/tools/indexing.html
@@ -365,7 +365,7 @@ 
Indexing Tools
 

-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1633458409.6711774.log')[source]

+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1635803414.020972.log')[source]
 
Verify all file object records into a manifest csv

 

 
Parameters

diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html
index 29a246fdf..707ab0953 100644
--- a/docs/_build/html/tools/metadata.html
+++ b/docs/_build/html/tools/metadata.html
@@ -102,7 +102,7 @@ 
Metadata Tools
 

-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1633458410.112426.log', get_guid_from_file=True, metadata_type=None)[source]

+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1635803414.470649.log', get_guid_from_file=True, metadata_type=None)[source]
 
Ingest all metadata records into a manifest csv

 

 
Parameters


From 775aac7d8048e43f0b6f1376b035084ac82fe281 Mon Sep 17 00:00:00 2001
From: Avantol13 
Date: Thu, 11 Nov 2021 22:50:49 +0000
Subject: [PATCH 3/3] Apply automatic documentation changes

---
 docs/_build/doctrees/environment.pickle     | Bin 226070 -> 226376 bytes
 docs/_build/doctrees/tools/indexing.doctree | Bin 93366 -> 93370 bytes
 docs/_build/doctrees/tools/metadata.doctree | Bin 34281 -> 34281 bytes
 docs/_build/html/searchindex.js             |   2 +-
 docs/_build/html/tools/indexing.html        |   2 +-
 docs/_build/html/tools/metadata.html        |   2 +-
 6 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle
index 14d07602a67d81a4bebcf711c3bb2d299313d2c8..da7944fa7575bb8f8232b0ba00a440d6b79c06fc 100644
GIT binary patch
delta 4444
zcmZ`+d303O8RxzkW`I)8lk+wbpw
z-(B7uzcKT7pUs?pKz;0x@xt`IiTl-Nmvg6E40O15G+P`Fi?ziqsv9g#yjz^*v|5`S
zS1s(9i2CKZ6l|wE
zRh*rfnbpX*+@D1&m}%SN?qbOAGSNaRCJ+2NIR>H(33(JIbqe#v`|7DUQry%W;xn^~
zDz;v*8FXX>x(x>HsNx5g4C$~md1H#-pVgT9{~Vz>AFddpAS^i?x{|ZBbfq{I+)7SQ
zzc-vI6#lGT!T`)kagZK3n(_twF;x#k^IUK;Lk}7EKLAas&9G{L9xkS7;H@Rma3@6z
zN%KF42=vQL+e?OF-@LhKZG)gRJ^Zym4~G|QgSK=%1gGzUnHdJ?OSi*0v}I>FVDCIV
zd|YLK&ogXnj|iM<+B8;WA)az{ZwP6Wqx(WwLOHr8gw>Rzg%;LRj@DGzK{>h)1RLe(
z9uW3Xj#g3-DMt$;^iqztT{wfBOH@0pbq@FRz3x<@NB$@?gFs5L0kX0uVbhWpTzmo?
z$T30gQUiRKGYPw^65#5>tFWQk03Y1{>V67Q)bcG(yM=eSx4Kg+1-gL_x+~c_=t!~C
zks_xf#Z5W4Yn-_9?mnt=G;&SSsn(r@+Ps3
z`wN;eWDXRqFhNkZ0bW@#2{)IwU`t!!%{&7fES!YjuC!uXqv72m6MVi}i7qQ1Urr(X
zPfchK=-Oxp?!7kFrG7|T)FzLW#1MF?+=hE44ysp~;5{QtF2Au#Pf)l|{&Dp|g>kQa
zk}@d}vStPp90-90YeHp9Sp~-DFNQD5&wzV{L7u3XLy!rDw6X};|9A}ZO0Gn&+u1F+
z1KZ%;90NR3eXWxAuAHzM<3O8R&anofMEi#WZ66M_;yBQn<3N{;16@S9nynJ!24J+N
zWF<|4`{EwW`#-%yf1V|zQQopFSQpPu2p(sK%6}JmCAuG`Y!vB~xLLJnjdkAg)T
z^jhJy0F--R>)~vK+;O-F>E*))s}#3Q@JPoz*xnJ2N>@iA(w93HBmJ_Y7-@E=*R2s=?VbnMI|BpU
z3K|c**ja=g|L9zyB(g!%5#}K!m`IRs&d0j)6mu)&Kom?q6b?7Lf)v#r_@Q%w)L|mp
zsiY5dQ*_R5I{8F*zT)^KoBY7`pkVLZv^-D4QbbgHG)(b1%+}#KitCVHJQ~fBL3rC;
zh;-JmQZ2iMFKD6rWDK+&n~&PzW5pV#oxmz*NTnCZECuo)q#a*{4z}Y3NXL&?BK?W+
z1rp^uq&%c=N)<}@CAiWP1wEFz-X;yg))OV@FnnSK((5N^5?#+iysS(;1xTT13DUQE
zRw51U%_F@~*;}S%m*OognESKgQg4KY$=|}qVie3Cij_XPO@e$S>V?d{QYGM@G$2tWoiso4(?RAZiN164@V2TPH*4w{huVXy{iyr)zNO@y4m9O(Cip>o_)
ztW28hn=~D@c|#P?Q$wkExWE1oTloeDNYAW5l%DZl+twL=$e0eexqlb|8%V(aO!Euv|W%<0CM5gN+d1q~1Lt|}&-P&A>n~XmE
z^2GRX#D2ZbX6GAtyUuCV)$Qc#cIg_dcAZ&QXKiV)G`j3O+L^nq&EnjtbM9n{R!co!
zuWKd4HCMd(M^WO_Gmu*?Gs#S_&XtSJx2Lm_OHSv()b5_b{E!?A$qb
z{<1)L=wfuVF8|)iq@?djSQ!xjBcSCFT3(4rgq`?-+!CvGx4)LR0*k{Ca_0DOn_w@=5YAY
z81q?<7N3psTS=r(Zoc(|nRujcs)>Q!o?@)D+el(HZuWLbRwJ3GrjLkJRY~*;x4oLo
zQ9>f5@2kjcQyVsN%cO>6-@CZbbNbXJgPh0Ck2|jbx@wh~478ypP_b+T_ot-y3c{lqskUlw2Qplk6*bBY{q2o;P
z`U@mV$)kg>rMc)rWRi$Ol8iV8r9v53Feq)75!~Q}OD5tR4gG4u7sIwIniGblq<2W9
znyRGnD;POEvE?0dwpmTz2+5qsCHc8Ht@2RxO4Ws261%>|IO)=D5=)Ct{c#t0`VvX6
z^vgodfWyayoL=#xhDhm7J~so^7#=S9UK38}mCA}ZJ?dXC;-Y=82un8?z8MUonjpA5
zpiw+hy%VD;QmW45w5Xn0%%v-8J*tzZJf@Jv!4Xdax}GcM;&7l4Aiq@$*+Vq%&0-2*
zVrB_fpT+*HI>cny{a83$e>ikvz|Oh0u)A4Ifh(s{$*5Fxk;_rjH$yuAHWx!iC%P_j
z6HC33dU*X8dg+ymTnN3zCwc-@JE};(^y5aAp7c-XHmR=1kO7IN}A_(t6~zEG5qcz
Wm#xWa#uGIlePUN7(y1M)i2ndqSLc=h

delta 4202
zcmZ`-dsI|)9`|==m_bm1a1`s!m!?Br^C{)*C}Zqsj*s|l1GiL+Oo$f
zdD$J?YHRHFnmQZm#JX*EJSrWHk2?UJ
z*gqglo3<+RgROzz?K-a)k56)Y?na3xCb_+Dzby7nay?%Zi2ajX|7YG}&m=cen=1BA
za_9GL5M7hp-&Y2S-ILt4KQ0spV<*{r^L@nO3GOc2dsZBp_
zG*S*p+gMj+cX~8AW5ogGBXc4H=W-JyC6B|-VjDIf91f(E!9yh`xSKK#HS2VY>f>y7Q1t^Gqy;ixi>qDcpY?7?sQV-bW+}QQr2`*&U8}7bW(23
zPKz|7##Uu@R9ooT;GubBwLQEa7%<4)2HpW)Jg<95iAzluFiAhyvG_K8n{0DWZ3kRQ
zF)5ptv#`SB&Fm_h+QSy__XD-e9B%P?B2)bF)m!8@Wb?gr9P6pAuKT-o5(N}jSZ
zRxpPlb05U7@rV6S86bX@pAu74z~I#d;m}_ECTv`+7WY`mheb5Ae8zi=2Msk};DbYs
z=7tB24G-!x9@J|*s8l>C`Ft&}4*iB;cx`qL1;OXxSBCa~W`_0_9+14;?T9Z8<8KDO
z>^!T<)#2voIhfK>rc<1Ihqx^%?DuM;X2VL#Q?Xh$q-<qT6o)9
zc#TN0u4=R%9&4WmTig9n
z=xASo_)_~K#M|vzh{+vpv+eLo*F3o1;pOR6bLoW(9hqqHUB_}YP(4`NxP@F?&b)Oz
z;a+&6GhJ2hf)ogX@rV84W~a9*+6(tO;$dcmr@OpfxW9{XYwV({$GVoOX20cEAK2^?Fdr2|K@
zbX+=y9Zr-VKhI{V3HLzU(OfjBKbnC!db9}fCr)R`M7PW7h!^Do)%`NO+Z_bmHPP-0
zdSLUhY%~}+wjA;LF$xmdy@2&WS$75^bf+P{)t!Ur*OSg%P}H+V&zIp>dZ_G6hRZzx
zI?jIsA4@^t(;p)H-euksU3Y=GH&1o=o*Wj-k=K}hmU|}L@gA4^)Hl?Lma4`H4fyu0
z!Rt2lrx;O-A>1Yfl|E@(YofzIj)6VM^OpK?0bAB%6-
zXn!_-1N^%`A92ON7DN}v9|zVVT{37vd}%NX@jrvBb$sXKLDX;>_&)C8elWPOfP{W!
zc^h-xrE_U9O8w#XkQx2MhTZ-TlIe^GOAE>-OyTTso*H`-cd$2^A>*V@6>Z_7ij!#=
zYdc^T(DSxk1ZvT32TziUy=T23s5q81FXCuJ+bTAa3*5e1wy3=Jk
z{)fJaTmR{Fp~@cMZ2WU;(WdFShhg-T2W#fCe?7MbWy{ZOMRc86#D?JgGiiuE&o4tP
zd_EJgi_su`Bpq?@$Wp`?M=}t<9eDspko-b8W?RJ3
z%JC55uyXo^>4M-)Rg~Xpnf$`$)tS|_YZy2=(Ru~8CZp3I4
zSYa+4xM3<-#R`kiR+>2(v&zw6RpS+wutFOtJtve-u>^NQzLkp|_<8hm)d1YN;jg8A
z3L9^P<9eNY(}aHk@^7Z-afvqFiDJhU*PW*;wX`$k`6VohuM{Z+eAkD1@Kr7k6tQ`#
z{>fryWWDk$r7Q}Mk4l+IwGM=L+JaFMRmO}PD3TOJ+2zr13UgJ1rLxgd(O{{nYp8be
zkUv*fTdbB}s(In6_jU`u#XO_|d^$qaJVuLIBwlx|jJXBL#VkxsB|t7IVgWkUM=mPj
znQkg)A)!3TeN%a1RmyQ
zQW=b&A7hmdvgXZN+NpBnc?{7bPghv@q#!`fP?#rQnvwyEkA|w6AI`BLH2saja4XAa
z6%0!uq-fass4;^7>6Cma-$M6qD>H8fw~2&1a#92L1(hW;`xX&-dz&k|x)n`rp@#BAEl-tf+G$^})yvdveRrNLX}
zuxue3g&EmGf@*I>`xuoBMKVN7$;mIPX4kTX(21)fbMWfqehSHNPRgO1$1-w+>Lfkw
zQo9rbJD>1}>wEmhF53miCcfLGx$y4ESS=aI8J7gJN_}!kFmNXaj&$b}(POtR3mr?`
zVT|zFlSUaOzO)j@E>72MD`vy8d4tBNr>*Faj8OTsaqRI8nlVEtZ8V7wb)CJev96(V
zyTx2phd)upA-T}531>s{4!b4_k2bp|h%Z1X05YEm^x(fz*)*k=AOPqLH8

diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree
index d887e1ebaa6d67186aa825606e4ad9ec52b5692d..0f7293159aed5f0ab101bf956b33da24858caed6 100644
GIT binary patch
delta 141
zcmdmXlXcfk)`lsJm5i2Ks~A(28Q*Sypv-uik>AYB+|aL6uQk234b8PJX)jlnnORDH&ot5EbY~Z!%=$um%7Lu_e#|

diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree
index 7ff3f8a55cf5d303828cad201d85583175fc327a..f40492e648e11db8484b92c082e84978db00e57e 100644
GIT binary patch
delta 88
zcmaFa&GfRHX~RJcJ~K0OLjx0YJtH$?b4%mR=QWI2(8c(5V_DI}CRZ6bqX}(3Z6p^A
E06vBm{r~^~

delta 88
zcmaFa&GfRHX~RJcK2r+=BLj0&JxgOV6C;Do=QWI2(8c(5V_DI}CRZ6bqX}(3Z6p^A
E063Ty)&Kwi

diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js
index 5a5d6abf6..8193fef60 100644
--- a/docs/_build/html/searchindex.js
+++ b/docs/_build/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0934136":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"112426":[],"11e9":9,"1394515":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1633103143":[],"1633373719":[],"1633449426":[],"1633449427":[],"1633454066":[],"1633454067":[],"1633458409":[],"1633458410":[],"1635802075":[9,10],"1731167":[],"197634":[],"2280114":[],"2343018":[],"255e396f":9,"2889535":[],"333":5,"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"480863":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5859108":[],"5964222":[],"622057":[],"641011":[],"6542356":[],"6572418":[],"6711774":[],"6f90":7,"7461076":[],"7649791":[],"772628":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"9159722":[],"9335642":[],"934012":[],"9364202":10,"93d9af72":9,"9644923":[],"9781935":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],use_agg_md:5,used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}})
\ No newline at end of file
+Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0934136":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"112426":[],"11e9":9,"1394515":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1633103143":[],"1633373719":[],"1633449426":[],"1633449427":[],"1633454066":[],"1633454067":[],"1633458409":[],"1633458410":[],"1635802075":[],"1636671046":9,"1636671047":10,"1731167":[],"197634":[],"2280114":[],"2343018":[],"255e396f":9,"2637932":10,"2889535":[],"333":5,"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"480863":[],"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5859108":[],"5964222":[],"622057":[],"641011":[],"6542356":[],"6572418":[],"6711774":[],"6f90":7,"7461076":[],"7649791":[],"772628":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"8849425":9,"9159722":[],"9335642":[],"934012":[],"9364202":[],"93d9af72":9,"9644923":[],"9781935":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],use_agg_md:5,used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}})
\ No newline at end of file
diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html
index fd59dc846..870bc6907 100644
--- a/docs/_build/html/tools/indexing.html
+++ b/docs/_build/html/tools/indexing.html
@@ -365,7 +365,7 @@ 
Indexing Tools
 

-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1635802075.480863.log')[source]

+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1636671046.8849425.log')[source]
 
Verify all file object records into a manifest csv

 

 
Parameters

diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html
index 8ef51ad0d..61b87b671 100644
--- a/docs/_build/html/tools/metadata.html
+++ b/docs/_build/html/tools/metadata.html
@@ -102,7 +102,7 @@ 
Metadata Tools
 

-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1635802075.9364202.log', get_guid_from_file=True, metadata_type=None)[source]

+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1636671047.2637932.log', get_guid_from_file=True, metadata_type=None)[source]
 
Ingest all metadata records into a manifest csv

 

 
Parameters