Merge pull request #70 from filak/dev-1.6.9

Dev 1.6.9

flask-app/!!build__dist_pyinstaller.bat

@@ -70,6 +70,16 @@ set extras=
 set subdir=tools\
 call:buildFiles
 
+set fileHandle=update-ns
+set extras=
+set subdir=tools\
+call:buildFiles
+
+set fileHandle=secrets-gen
+set extras=
+set subdir=tools\
+call:buildFiles
+
 echo.
 echo Finished !!!
 echo.

flask-app/.vscode/launch.json

@@ -17,7 +17,7 @@
                 //"--port=80",
                 "--port=5900",
                 "--debugger",
-                "--reload" 
+                "--reload"
             ],
             "jinja": true
         },
@@ -54,8 +54,8 @@
                 //"--port", "80",
                 //"--config", "conf/mtw.ini",
                 //"--config", "conf/mtw-fr.ini",
-                //"--debug",
-                "--fqdn=test.medvik.cz", 
+                "--debug",
+                "--fqdn=test.medvik.cz",
                 //"--relax"
             ],
             "jinja": true
@@ -74,11 +74,11 @@
                 //"--port", "5903",
                 //"--config", "conf/mtw.ini",
                 //"--config", "conf/mtw-fr.ini",
-                //"--debug"
+                "--debug"
                 //"--relax"
             ],
             "jinja": true
-        },                
+        },
         {
             "name": "mesh-trx2nt",
             "type": "debugpy",
@@ -90,7 +90,7 @@
                 "tools/trans_only_2023_expanded_fre.txt",
                 "fr",
                 "http://meshfr.medvik.cz/link/"
-            ],            
-        }        
+            ],
+        }
     ]
 }

flask-app/application/main.py

@@ -56,7 +56,7 @@ def create_app(debug=False, logger=None, port=5900,
     app.config.update(dict(
         APPLICATION_ROOT = url_prefix,
         APP_NAME = 'MTW',
-        APP_VER = '1.6.8',
+        APP_VER = '1.6.9',
         API_VER = '1.0.0',
         DBVERSION = 1.0,
         CACHE_DIR = mtu.get_instance_dir(app, 'cache'),

flask-app/application/modules/routes.py

@@ -1072,12 +1072,18 @@ def search(dui, action):
         if scr in ['yes', 'no']:
             session['scr'] = scr
 
+    if request.args.get('qtp'):
+        qtp = request.args.get('qtp').strip()
+        if qtp in ['def', 'all', 'notes']:
+            session['qtp'] = qtp
+
     if action == 'clear':
         session.pop('q', None)
         session.pop('sshow', None)
         session.pop('sstatus', None)
         session.pop('slang', None)
         session.pop('scr', None)
+        session.pop('qtp', None)
         session.pop('dui', None)
         session.pop('adui', None)
 
@@ -1150,7 +1156,7 @@ def search(dui, action):
     if text_query:
         data = sparql.getSparqlData('search',
                                     query=text_query, show=session.get('sshow'), status=session.get('sstatus'),
-                                    slang=session.get('slang'), scr=session.get('scr'))
+                                    slang=session.get('slang'), scr=session.get('scr'), qtp=session.get('qtp'))
         # pp.pprint(data)
 
         if data:
@@ -1180,7 +1186,8 @@ def search(dui, action):
 
     return render_template('search.html',
                            hits_cnt=hits_cnt, hits=hits, dui=dui, tree=tree, descriptor=descriptor, concepts=concepts, tab=tab,
-                           show=session.get('sshow'), status=session.get('sstatus'), slang=session.get('slang'), scr=session.get('scr'),
+                           show=session.get('sshow'), status=session.get('sstatus'),
+                           slang=session.get('slang'), scr=session.get('scr'), qtp=session.get('qtp'),
                            audit=audit, dview=dview)
 
 

flask-app/application/modules/sparql.py

@@ -36,7 +36,7 @@ def show_elapsed(begin, tag=''):
 
 
 def getSparqlData(template, query='', show='', status='', top='', tn='', concept='', otype='Descriptor',
-                  output='json', slang=None, lang=None, scr=None, key=None, cache=None):
+                  output='json', slang=None, lang=None, scr=None, qtp=None, key=None, cache=None):
 
     t0 = timer()
 
@@ -60,9 +60,10 @@ def getSparqlData(template, query='', show='', status='', top='', tn='', concept
 
     sparql = render_template('sparql/' + template + '.sparql', query=cleanQuery(query), otype=otype,
                              show=show, status=status, top=top, tn=tn, toptn=toptn, concept=concept,
-                             lang=lang, lang_umls=lang_umls, slang=slang, scr=scr)
+                             lang=lang, lang_umls=lang_umls, slang=slang, scr=scr, qtp=qtp)
 
-    # print(sparql)
+    if app.debug:
+        print(sparql, '\n')
 
     endpoint = app.config['SPARQL_HOST'] + app.config['SPARQL_DATASET'] + '/query'
 

flask-app/application/modules/utils.py

@@ -156,6 +156,7 @@ def getLocalConfValue(conf, fp=''):
         d['GCSP'] = json.loads(conf.get(section, 'GCSP'))
         d['CHAR_NORM_FILE'] = conf.get(section, 'CHAR_NORM_FILE', fallback='norm_chars_table.tsv.txt')
         d['MESH_BROWSER'] = conf.get(section, 'MESH_BROWSER', fallback='https://meshb.nlm.nih.gov/record/ui?ui=')
+        d['USE_PROP_LISTS'] = conf.getboolean(section, 'USE_PROP_LISTS', fallback=False)
 
         section = 'sparqlconf'
         d['SPARQL_HOST'] = conf.get(section, 'SPARQL_HOST', fallback='http://127.0.0.1:3030/')
@@ -349,7 +350,7 @@ def getLookupJson(lookups, export):
         dui = item['dui']
 
         d = {}
-        if item.get('active', '') == 'false':
+        if item.get('active') == 'false':
             d['active'] = False
             d['eng'] = item.get('den', '').replace('[OBSOLETE]', '').strip()
         else:
@@ -378,7 +379,7 @@ def getLookupJson(lookups, export):
 
         if export == 'marc':
             if xterms.get(dui):
-                if xterms[dui].get('active') == 'true':
+                if xterms[dui].get('active', 'true') == 'true':
                     # terms termsx nterms ntermsx
                     for termtype in ['terms', 'termsx', 'nterms', 'ntermsx']:
                         if xterms[dui].get(termtype):
@@ -396,7 +397,7 @@ def getLookupJson(lookups, export):
             trees = item.get('trn', '').split('~')
             for trn in trees:
                 if trn:
-                    if item.get('active', '') == 'false':
+                    if item.get('active') == 'false':
                         trn = trn.replace('[OBSOLETE]', '').strip()
                         if 'x' not in d['cat']:
                             d['cat'].append('x')
@@ -557,7 +558,7 @@ def getElasticData(data):
                 # resp.append({'index': {'_id': trn, '_index': 'mesht'}})
                 resp.append({'id': dui, 'db': 'mesht', 'trn': trn.replace('.', '-'),
                              'eng': item.get('eng', ''), 'trx': item.get('trx', ''),
-                             'active': item.get('active')})
+                             'active': item.get('active', 'true')})
 
         # Add later by running:  grind-data elastic mesh ...
         # resp.append({'index': {'_id': dui, '_index': 'mesh'}})
@@ -579,7 +580,7 @@ def getElasticData(data):
                 # resp.append({'index': {'_id': trn, '_index': 'mesht'}})
                 resp.append({'id': dui, 'db': 'mesht', 'trn': trn.replace('.', '-'),
                              'eng': item.get('eng', ''), 'trx': item.get('trx', ''),
-                             'active': item.get('active')})
+                             'active': item.get('active', 'true')})
 
         if qualifs.get(dui):
             qa = []
@@ -760,7 +761,7 @@ def getLookupXml(lookups, export, as_string=True):
         trx = html.escape(items[k].get('trx', ''))
         dclass = items[k].get('dc', '-1')
 
-        if items[k].get('active') is False:
+        if items[k].get('active', True) is False:
             line = '<%s id="%s" eng="%s" trx="%s" dclass="%s" active="0" />\n' % (tag, k, eng, trx, dclass)
         else:
             line = '<%s id="%s" eng="%s" trx="%s" dclass="%s" />\n' % (tag, k, eng, trx, dclass)
@@ -827,7 +828,7 @@ def getMarc(lookups, export, params, as_string=True):
 
         item = items[mid]
 
-        if item.get('active') is True:
+        if item.get('active', True):
             cnt += 1
             marc.write(leader)
             marc.write(line_pref + '001' + code_pref + mid + '\n')
@@ -879,7 +880,7 @@ def getMarc(lookups, export, params, as_string=True):
             marc.write('\n')
 
         qa = qualifs.get(mid)
-        if item.get('active') is True and qa and ecin == 'yes':
+        if item.get('active', True) and qa and ecin == 'yes':
             for qui in qa.split('~'):
                 cnt += 1
                 marc.write(leader)
@@ -948,7 +949,7 @@ def getMarcFields(dui, item, descriptors, qualifiers, qualifs, xnote, lp='=', cp
     btdx = []
     for d in item.get('btd', []):
         des = descriptors.get(d)
-        if des.get('active') is True:
+        if des.get('active', True) is True:
             trx = des.get('trx')
             if not trx:
                 trx = des.get('eng')
@@ -964,7 +965,7 @@ def getMarcFields(dui, item, descriptors, qualifiers, qualifs, xnote, lp='=', cp
     ntdx = []
     for d in item.get('ntd', []):
         des = descriptors.get(d)
-        if des.get('active') is True:
+        if des.get('active', True) is True:
             trx = des.get('trx')
             if not trx:
                 trx = des.get('eng')
@@ -980,7 +981,7 @@ def getMarcFields(dui, item, descriptors, qualifiers, qualifs, xnote, lp='=', cp
     rtdx = []
     for d in item.get('rtd', []):
         des = descriptors.get(d)
-        if des.get('active') is True:
+        if des.get('active', True) is True:
             trx = des.get('trx')
             if not trx:
                 trx = des.get('eng')
@@ -996,7 +997,7 @@ def getMarcFields(dui, item, descriptors, qualifiers, qualifs, xnote, lp='=', cp
     pax = []
     for d in item.get('pa', []):
         des = descriptors.get(d)
-        if des.get('active') is True:
+        if des.get('active', True) is True:
             trx = des.get('trx')
             if not trx:
                 trx = des.get('eng')
@@ -1409,7 +1410,7 @@ def getParamsForAudit(dui, concept, cui):
     par['cui'] = cui
 
     if citem:
-        par['active'] = citem['active']
+        par['active'] = citem.get('active', True)
         par['label'] = citem['label']
         par['rel'] = citem['rel']
         par['terms'] = citem['terms']['target']

flask-app/application/templates/modals/edit-concept.html

@@ -15,7 +15,7 @@ <h4 class="modal-title">{{ descriptor['labels']['en'] }}
 
      {% if item['ntx'] %}
          {% set text_notrx_label = 'text-warning' %}
-     {% elif item['active'] == 'false' %}
+     {% elif item.get('active') == 'false' %}
          {% set text_notrx_label = 'text-danger' %}
      {% endif %}
 
@@ -167,7 +167,7 @@ <h4 class="modal-title">
 
           <input type="hidden"
                  name="active"
-                 value="{{ item['active'] }}">
+                 value="{{ item.get('active', 'true') }}">
 
           <input type="hidden"
                  name="label"
@@ -216,7 +216,7 @@ <h4 class="modal-title">
 
           {% if session.ugroup in ['admin','manager','editor'] %}
 
-            {% if item['active'] == 'false' %}
+            {% if item.get('active') == 'false' %}
               {% if session.ugroup in ['admin','manager'] %}
                {% set del_butt_label = 'Purge Concept & Terms' %}
                {% set del_butt_outline = 'danger' %}
@@ -237,10 +237,11 @@ <h4 class="modal-title">
 
             <button type="button" class="btn btn-outline-{{ del_butt_outline }} mr-auto {{ del_butt_invisible }} {{ del_butt_disabled }}"
                     data-dismiss="modal" data-toggle="modal" data-target="#deleteConcept" data-concept="{{ item['cui'] }}"
-                    data-form="#editConcept-{{ item['cui'] }}" data-active="{{ item['active'] }}">{{ del_butt_label }}</button>
+                    data-form="#editConcept-{{ item['cui'] }}"
+                    data-active="{{ item.get('active', 'true') }}">{{ del_butt_label }}</button>
 
           {% else %}
-            {% if item['active'] == 'false' %}
+            {% if item.get('active') == 'false' %}
                <button type="button" class="btn btn-outline-warning mr-auto">Deleted</button>
             {% endif %}
           {% endif %}
@@ -249,7 +250,7 @@ <h4 class="modal-title">
 
             <button type="button" class="btn btn-link ml-3" data-dismiss="modal">Close</button>
 
-         {% if item['active'] == 'false' and item['lang'] == config.TARGET_LANG and item['lang'] != 'en' %}
+         {% if item.get('active') == 'false' and item['lang'] == config.TARGET_LANG and item['lang'] != 'en' %}
             {% set submit_inactive = 'disabled' %}
          {% endif %}
 

flask-app/application/templates/snips/concepts.html

@@ -12,7 +12,7 @@
       <ul class="list-unstyled {{ c_class }}">
           {% if item['ntx'] %}
             {% set item_text = 'text-warning' %}
-          {% elif item['active'] == 'false'  %}
+          {% elif item.get('active') == 'false' %}
             {% set item_text = 'text-danger' %}
           {% else %}
             {% set item_text = 'text-muted' %}
@@ -33,7 +33,7 @@
 
           {% elif item['status'] == 'OK' %}
 
-            {% if not item.get('cpid') and item.get('tnote') and session.ugroup in ['admin'] and item.get('active') == 'true' %}
+            {% if not item.get('cpid') and item.get('tnote') and session.ugroup in ['admin'] and item.get('active', 'true') == 'true' %}
 
             <a href="{{ url_for('add_cpid', dui=dui, cui=item['cui']) }}">
               <button type="link" class="btn btn-primary btn-sm ml-1" title="Generate custom CUI">
@@ -53,31 +53,36 @@
           {% if item['scn'] and item['scnt'] %}
           <button type="button" class="btn btn-primary btn-sm ml-1" title="Edit ScopeNote"
                   data-toggle="modal" data-target="#editScopeNote"
-                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}" data-active="{{ item['active'] }}"
+                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}"
+                  data-active="{{ item.get('active', 'true') }}"
                   data-scn="{{ item['scn'] }}" data-scnt="{{ item['scnt'] }}">
             <i class="fas fa-sticky-note text-success"></i></button>
           {% elif item['scn'] %}
           <button type="button" class="btn btn-primary btn-sm ml-1" title="Add Translated ScopeNote"
                   data-toggle="modal" data-target="#editScopeNote"
-                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}" data-active="{{ item['active'] }}"
+                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}"
+                  data-active="{{ item.get('active', 'true') }}"
                   data-scn="{{ item['scn'] }}" data-scnt="">
             <i class="fas fa-sticky-note text-info"></i></button>
           {% elif item['scne'] and item['scnt'] %}
           <button type="button" class="btn btn-primary btn-sm ml-1" title="Edit ScopeNotes"
                   data-toggle="modal" data-target="#editScopeNote" data-enable="scn"
-                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}" data-active="{{ item['active'] }}"
+                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}"
+                  data-active="{{ item.get('active', 'true') }}"
                   data-scne="{{ item['scne'] }}" data-scnt="{{ item['scnt'] }}">
             <i class="fas fa-sticky-note text-success"></i></button>
           {% elif item['scnt']  %}
           <button type="button" class="btn btn-primary btn-sm ml-1" title="Add English ScopeNote"
                   data-toggle="modal" data-target="#editScopeNote" data-enable="scn"
-                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}" data-active="{{ item['active'] }}"
+                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}"
+                  data-active="{{ item.get('active', 'true') }}"
                   data-scne="" data-scnt="{{ item['scnt'] }}">
             <i class="fas fa-sticky-note text-warning"></i></button>
           {% elif item['scne']  %}
           <button type="button" class="btn btn-primary btn-sm ml-1" title="Add Translated ScopeNote"
                   data-toggle="modal" data-target="#editScopeNote" data-enable="scn"
-                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}" data-active="{{ item['active'] }}"
+                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}"
+                  data-active="{{ item.get('active', 'true') }}"
                   data-scne="{{ item['scne'] }}" data-scnt="">
             <i class="fas fa-sticky-note text-info"></i></button>
 
@@ -86,7 +91,8 @@
             {% if item['status'] == 'OK' and item['lang'] == config.TARGET_LANG %}
             <button type="button" class="btn btn-primary btn-sm ml-1" title="Add ScopeNotes"
                   data-toggle="modal" data-target="#editScopeNote" data-enable="scn"
-                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}" data-active="{{ item['active'] }}"
+                  data-cpid="{{ item['cpid'] }}" data-label="{{ item['label']|truncate(50, True) }}" data-concept="{{ item['rc'] }}"
+                  data-active="{{ item.get('active', 'true') }}"
                   data-scne="" data-scnt="">
             <i class="fas fa-sticky-note"></i></button>
             {% endif %}