Skip to content

feat(profiles): simplify configuration and use standard .ini library,… #103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Avantol13 wants to merge 4 commits into
base: master
Choose a base branch
from
Open

feat(profiles): simplify configuration and use standard .ini library,… #103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b6ff6e5
feat(profiles): simplify configuration and use standard .ini library,…
Avantol13-machine-user Nov 1, 2021
89e5c20
Apply automatic documentation changes
Avantol13 Nov 1, 2021
0753a7b
Merge branch 'master' into feat/profile
Avantol13-machine-user Nov 11, 2021
775aac7
Apply automatic documentation changes
Avantol13 Nov 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified docs/_build/doctrees/environment.pickle
View file
Open in desktop
Binary file not shown.
Binary file modified docs/_build/doctrees/tools/indexing.doctree
View file
Open in desktop
Binary file not shown.
Binary file modified docs/_build/doctrees/tools/metadata.doctree
View file
Open in desktop
Binary file not shown.
9 changes: 7 additions & 2 deletions docs/_build/html/_modules/gen3/auth.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ <h1>Source code for gen3.auth</h1><div class="highlight"><pre>
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
<span class="kn">import</span> <span class="nn">backoff</span>

<span class="kn">from</span> <span class="nn">gen3.utils</span> <span class="kn">import</span> <span class="n">DEFAULT_BACKOFF_SETTINGS</span><span class="p">,</span> <span class="n">raise_for_status</span>
<span class="kn">from</span> <span class="nn">gen3.utils</span> <span class="kn">import</span> <span class="n">DEFAULT_BACKOFF_SETTINGS</span><span class="p">,</span> <span class="n">raise_for_status</span><span class="p">,</span> <span class="n">get_cfg_from_profile</span>


<span class="k">class</span> <span class="nc">Gen3AuthError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
Expand Down Expand Up @@ -197,7 +197,12 @@ <h1>Source code for gen3.auth</h1><div class="highlight"><pre>
<span class="k">if</span> <span class="ow">not</span> <span class="n">refresh_file</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">refresh_token</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">idp</span><span class="p">:</span>
<span class="n">refresh_file</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">getenv</span><span class="p">(</span><span class="s2">&quot;GEN3_API_KEY&quot;</span><span class="p">,</span> <span class="s2">&quot;credentials&quot;</span><span class="p">)</span>

<span class="k">if</span> <span class="n">refresh_file</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">idp</span><span class="p">:</span>
<span class="c1"># prefer to check for a configure profile with the provided name first</span>
<span class="k">if</span> <span class="n">get_cfg_from_profile</span><span class="p">(</span><span class="n">profile</span><span class="o">=</span><span class="n">refresh_file</span><span class="p">,</span> <span class="n">cfg</span><span class="o">=</span><span class="s2">&quot;api_key_filepath&quot;</span><span class="p">):</span>
<span class="n">refresh_file</span> <span class="o">=</span> <span class="n">get_cfg_from_profile</span><span class="p">(</span>
<span class="n">profile</span><span class="o">=</span><span class="n">refresh_file</span><span class="p">,</span> <span class="n">cfg</span><span class="o">=</span><span class="s2">&quot;api_key_filepath&quot;</span>
<span class="p">)</span>
<span class="k">elif</span> <span class="n">refresh_file</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">idp</span><span class="p">:</span>
<span class="n">idp_prefix</span> <span class="o">=</span> <span class="s2">&quot;idp://wts/&quot;</span>
<span class="n">access_token_prefix</span> <span class="o">=</span> <span class="s2">&quot;accesstoken:///&quot;</span>
<span class="k">if</span> <span class="n">refresh_file</span><span class="p">[</span><span class="mi">0</span> <span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">idp_prefix</span><span class="p">)]</span> <span class="o">==</span> <span class="n">idp_prefix</span><span class="p">:</span>
Expand Down
2 changes: 1 addition & 1 deletion docs/_build/html/searchindex.js
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion docs/_build/html/tools/indexing.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ <h1>Indexing Tools<a class="headerlink" href="#indexing-tools" title="Permalink

<dl class="py function">
<dt id="gen3.tools.indexing.verify_manifest.async_verify_object_manifest">
<em class="property"><span class="pre">async</span> </em><code class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></code><code class="sig-name descname"><span class="pre">async_verify_object_manifest</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="pre">commons_url</span></em>, <em class="sig-param"><span class="pre">manifest_file</span></em>, <em class="sig-param"><span class="pre">max_concurrent_requests=24</span></em>, <em class="sig-param"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></em>, <em class="sig-param"><span class="pre">manifest_file_delimiter=None</span></em>, <em class="sig-param"><span class="pre">output_filename='verify-manifest-errors-1635802075.480863.log'</span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition">¶</a></dt>
<em class="property"><span class="pre">async</span> </em><code class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></code><code class="sig-name descname"><span class="pre">async_verify_object_manifest</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="pre">commons_url</span></em>, <em class="sig-param"><span class="pre">manifest_file</span></em>, <em class="sig-param"><span class="pre">max_concurrent_requests=24</span></em>, <em class="sig-param"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></em>, <em class="sig-param"><span class="pre">manifest_file_delimiter=None</span></em>, <em class="sig-param"><span class="pre">output_filename='verify-manifest-errors-1636671046.8849425.log'</span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition">¶</a></dt>
<dd><p>Verify all file object records into a manifest csv</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
Expand Down
2 changes: 1 addition & 1 deletion docs/_build/html/tools/metadata.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ <h1>Metadata Tools<a class="headerlink" href="#metadata-tools" title="Permalink

<dl class="py function">
<dt id="gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest">
<em class="property"><span class="pre">async</span> </em><code class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></code><code class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="pre">commons_url</span></em>, <em class="sig-param"><span class="pre">manifest_file</span></em>, <em class="sig-param"><span class="pre">metadata_source</span></em>, <em class="sig-param"><span class="pre">auth=None</span></em>, <em class="sig-param"><span class="pre">max_concurrent_requests=24</span></em>, <em class="sig-param"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></em>, <em class="sig-param"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></em>, <em class="sig-param"><span class="pre">manifest_file_delimiter=None</span></em>, <em class="sig-param"><span class="pre">output_filename='ingest-metadata-manifest-errors-1635802075.9364202.log'</span></em>, <em class="sig-param"><span class="pre">get_guid_from_file=True</span></em>, <em class="sig-param"><span class="pre">metadata_type=None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition">¶</a></dt>
<em class="property"><span class="pre">async</span> </em><code class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></code><code class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="pre">commons_url</span></em>, <em class="sig-param"><span class="pre">manifest_file</span></em>, <em class="sig-param"><span class="pre">metadata_source</span></em>, <em class="sig-param"><span class="pre">auth=None</span></em>, <em class="sig-param"><span class="pre">max_concurrent_requests=24</span></em>, <em class="sig-param"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></em>, <em class="sig-param"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></em>, <em class="sig-param"><span class="pre">manifest_file_delimiter=None</span></em>, <em class="sig-param"><span class="pre">output_filename='ingest-metadata-manifest-errors-1636671047.2637932.log'</span></em>, <em class="sig-param"><span class="pre">get_guid_from_file=True</span></em>, <em class="sig-param"><span class="pre">metadata_type=None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition">¶</a></dt>
<dd><p>Ingest all metadata records into a manifest csv</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
Expand Down
9 changes: 7 additions & 2 deletions gen3/auth.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from urllib.parse import urlparse
import backoff

from gen3.utils import DEFAULT_BACKOFF_SETTINGS, raise_for_status
from gen3.utils import DEFAULT_BACKOFF_SETTINGS, raise_for_status, get_cfg_from_profile


class Gen3AuthError(Exception):
Expand Down Expand Up @@ -164,7 +164,12 @@ def __init__(self, endpoint=None, refresh_file=None, refresh_token=None, idp=Non
if not refresh_file and not refresh_token and not idp:
refresh_file = os.getenv("GEN3_API_KEY", "credentials")

if refresh_file and not idp:
# prefer to check for a configure profile with the provided name first
if get_cfg_from_profile(profile=refresh_file, cfg="api_key_filepath"):
refresh_file = get_cfg_from_profile(
profile=refresh_file, cfg="api_key_filepath"
)
elif refresh_file and not idp:
idp_prefix = "idp://wts/"
access_token_prefix = "accesstoken:///"
if refresh_file[0 : len(idp_prefix)] == idp_prefix:
Expand Down
5 changes: 2 additions & 3 deletions gen3/cli/configure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@ def configure(profile, cred):
logging.info(f"Configuring profile [ {profile} ] with credentials at {cred}")

try:
profile_title, new_lines = config_tool.get_profile_from_creds(profile, cred)
lines = config_tool.get_current_config_lines()
config_tool.update_config_lines(lines, profile_title, new_lines)
cfg = config_tool.get_profile_from_creds(cred)
config_tool.update_config_lines(profile, cfg)
except Exception as e:
logging.warning(str(e))
raise e
52 changes: 18 additions & 34 deletions gen3/configure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,58 +19,42 @@

"""
import json
import logging
from os.path import expanduser
from cdislogging import get_logger
import configparser
from os.path import expanduser, abspath
from pathlib import Path
from collections import OrderedDict
import gen3.auth as auth_tool

import gen3.auth as auth_tool
from gen3.utils import CONFIG_FILE_PATH

CONFIG_FILE_PATH = expanduser("~/.gen3/config")
logging = get_logger("__name__")


def get_profile_from_creds(profile, cred):
def get_profile_from_creds(cred):
with open(expanduser(cred)) as f:
creds_from_json = json.load(f)
credentials = OrderedDict()
credentials["key_id"] = creds_from_json["key_id"]
credentials["api_key"] = creds_from_json["api_key"]
credentials["api_key_filepath"] = abspath(f.name)
credentials["api_endpoint"] = auth_tool.endpoint_from_token(
credentials["api_key"]
)
credentials["access_key"] = auth_tool.get_access_token_with_key(credentials)
credentials["use_shepherd"] = ""
credentials["min_shepherd_version"] = ""
profile_line = "[" + profile + "]\n"
new_lines = [key + "=" + value + "\n" for key, value in credentials.items()]
new_lines.append("\n") # Adds an empty line between two profiles.
return profile_line, new_lines


def get_current_config_lines():
""" Read lines from the config file if exists in ~/.gen3 folder, else create new config file """
try:
with open(CONFIG_FILE_PATH) as configFile:
logging.info(f"Reading existing config file at {CONFIG_FILE_PATH}")
return configFile.readlines()
except FileNotFoundError:
Path(CONFIG_FILE_PATH).touch()
logging.info(f"Config file doesn't exist at {CONFIG_FILE_PATH}, creating one")
return []
return credentials


def update_config_lines(lines, profile_title, new_lines):
""" Update config file contents with the new profile values """
def update_config_lines(profile, creds):
"""
Update config file contents with the new profile values
"""
config = configparser.ConfigParser()
config.read(CONFIG_FILE_PATH)

if profile_title in lines:
profile_line_index = lines.index(profile_title)
next_profile_index = len(lines)
for i in range(profile_line_index, len(lines)):
if lines[i][0] == "[":
next_profile_index = i
break
del lines[profile_line_index:next_profile_index]
config[profile] = creds

with open(CONFIG_FILE_PATH, "a+") as configFile:
configFile.write(profile_title)
configFile.writelines(new_lines)
with open(CONFIG_FILE_PATH, "w") as file:
config.write(file)
22 changes: 22 additions & 0 deletions gen3/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@
import sys
import re
import requests
import configparser
from os.path import expanduser

from urllib.parse import urlunsplit
from urllib.parse import urlencode
from urllib.parse import urlsplit
from urllib.parse import parse_qs

CONFIG_FILE_PATH = expanduser("~/.gen3/config")
Copy link
Contributor

@mfshao mfshao Nov 11, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fyi gen3-client has also been updated recently to adpot a standard .ini library for its config files, and the config file name has been changed in there https://github.com/uc-cdis/cdis-data-client/pull/104/files#diff-bd2e7cbbf1baffd3573ff07a2de731e31dca5ef335d98bd695d8ae4baea99c5eR26


UUID_FORMAT = (
r"^.*[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$"
Expand Down Expand Up @@ -166,6 +169,25 @@ def get_urls(raw_urls_string):
]


def get_cfg_from_profile(profile, cfg):
"""
Return value of `cfg` in configure profile `profile` or None if either don't exist
Args:
profile (str): name of profile previously setup using `gen3 configure`
cfg (str): name of configuration variable within the profile

Returns:
str: Value of `cfg` in provided configure profil
"""
config = configparser.ConfigParser()
config.read(CONFIG_FILE_PATH)

try:
return config[profile][cfg]
except Exception as exc:
return None


# Default settings to control usage of backoff library.
DEFAULT_BACKOFF_SETTINGS = {
"on_backoff": log_backoff_retry,
Expand Down