This repository has been archived by the owner on Sep 16, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 5
/
GSASIIpath.py
2607 lines (2389 loc) · 99.3 KB
/
GSASIIpath.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# -*- coding: utf-8 -*-
#GSASIIpath - file location & update routines
########### SVN repository information ###################
# $Date: 2024-03-17 12:50:24 -0500 (Sun, 17 Mar 2024) $
# $Author: toby $
# $Revision: 5767 $
# $URL: https://subversion.xray.aps.anl.gov/pyGSAS/trunk/GSASIIpath.py $
# $Id: GSASIIpath.py 5767 2024-03-17 17:50:24Z toby $
########### SVN repository information ###################
'''
:mod:`GSASIIpath` Classes & routines follow
'''
from __future__ import division, print_function
import os
import sys
import platform
import glob
import subprocess
import datetime as dt
try:
import numpy as np
except ImportError:
print("skipping numpy in GSASIIpath")
try:
import requests
try:
import git
except ImportError as msg:
if 'Failed to initialize' in msg.msg:
print('The gitpython package is unable to locate a git installation.')
print('See https://gsas-ii.readthedocs.io/en/latest/packages.html for more information.')
elif 'No module' in msg.msg:
print('Python gitpython module not installed')
else:
print(f'gitpython failed to import, but why? Error:\n{msg}')
except Exception as msg:
print(f'git import failed with unexpected error:\n{msg}')
except:
print('Python requests package not installed (required for GSAS-II\n'+
'to install/update from git)')
path2GSAS2 = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) # location of this file; save before any changes in pwd
# convert version numbers as '1.2.3' to integers (1002) and back (to 1.2)
fmtver = lambda v: str(v//1000)+'.'+str(v%1000)
intver = lambda vs: sum([int(i) for i in vs.split('.')[0:2]]*np.array((1000,1)))
def GetConfigValue(key,default=None):
'''Return the configuration file value for key or a default value if not present
:param str key: a value to be found in the configuration (config.py) file
:param default: a value to be supplied if none is in the config file or
the config file is not found. Defaults to None
:returns: the value found or the default.
'''
try:
return configDict.get(key,default)
except NameError: # this happens when building docs
return None
def SetConfigValue(parmdict):
'''Set configuration variables from a dictionary where elements are lists
First item in list is the default value and second is the value to use.
'''
global configDict
for var in parmdict:
if var in configDict:
del configDict[var]
if isinstance(parmdict[var],tuple):
configDict[var] = parmdict[var]
else:
if parmdict[var][1] is None: continue
if parmdict[var][1] == '': continue
if parmdict[var][0] == parmdict[var][1]: continue
configDict[var] = parmdict[var][1]
def addPrevGPX(fil,configDict):
'''Add a GPX file to the list of previous files.
Move previous names to start of list. Keep most recent five files
'''
fil = os.path.abspath(os.path.expanduser(fil))
if 'previous_GPX_files' not in configDict: return
try:
pos = configDict['previous_GPX_files'][1].index(fil)
if pos == 0: return
configDict['previous_GPX_files'][1].pop(pos) # if present, remove if not 1st
except ValueError:
pass
except AttributeError:
configDict['previous_GPX_files'][1] = []
files = list(configDict['previous_GPX_files'][1])
files.insert(0,fil)
configDict['previous_GPX_files'][1] = files[:5]
# routines for looking a version numbers in files
version = -1
def SetVersionNumber(RevString):
'''Set the subversion (svn) version number
:param str RevString: something like "$Revision: 5767 $"
that is set by subversion when the file is retrieved from subversion.
Place ``GSASIIpath.SetVersionNumber("$Revision: 5767 $")`` in every python
file.
'''
try:
RevVersion = int(RevString.split(':')[1].split()[0])
global version
version = max(version,RevVersion)
except:
pass
def LoadConfigFile(filename):
'''Read a GSAS-II configuration file.
Comments (starting with "%") are removed, as are empty lines
:param str filename: base file name (such as 'file.dat'). Files with this name
are located from the path and the contents of each are concatenated.
:returns: a list containing each non-empty (after removal of comments) line
found in every matching config file.
'''
info = []
for path in sys.path:
fil = os.path.join(path,'inputs',filename)
if not os.path.exists(fil): # patch 3/2024 for svn dir organization
fil = os.path.join(path,filename)
if not os.path.exists(fil): continue
try:
i = 0
fp = open(fil,'r')
for line in fp:
expr = line.split('#')[0].strip()
if expr:
info.append(expr)
i += 1
print(str(i)+' lines read from config file '+fil)
finally:
fp.close()
return info
def GetBinaryPrefix(pyver=None):
'''Creates the first part of the binary directory name
such as linux_64_p3.9 (where the full name will be
linux_64_p3.9_n1.21).
Note that any change made here is also needed in GetBinaryDir in
fsource/SConstruct
'''
if sys.platform == "win32":
prefix = 'win'
elif sys.platform == "darwin":
prefix = 'mac'
elif sys.platform.startswith("linux"):
prefix = 'linux'
else:
print(u'Unknown platform: '+sys.platform)
raise Exception('Unknown platform')
if 'arm' in platform.machine() and sys.platform == "darwin":
bits = 'arm'
elif 'aarch' in platform.machine() and '64' in platform.architecture()[0]:
bits = 'arm64'
elif 'arm' in platform.machine():
bits = 'arm32'
elif '64' in platform.architecture()[0]:
bits = '64'
else:
bits = '32'
# format current python version
if pyver:
pyver = 'p'+pyver
else:
pyver = 'p{}.{}'.format(*sys.version_info[0:2])
return '_'.join([prefix,bits,pyver])
#==============================================================================
#==============================================================================
# hybrid routines that use git & svn (to be revised to remove svn someday)
G2_installed_result = None
def HowIsG2Installed():
'''Determines if GSAS-II was installed with git, svn or none of the above.
Result is cached to avoid time needed for multiple calls of this.
:returns:
* a string starting with 'git' from git,
if installed from the GSAS-II GitHub repository (defined in g2URL),
the string is 'github', if the post-3/2024 directory structure is
in use '-rev' is appended.
* or 'svn' is installed from an svn repository (assumed as defined in g2home)
* or 'noVCS' if installed without a connection to a version control system
'''
global G2_installed_result
if G2_installed_result is not None: return G2_installed_result
try:
g2repo = openGitRepo(path2GSAS2)
if os.path.samefile(os.path.dirname(g2repo.common_dir),path2GSAS2):
rev = ''
else:
rev = '-rev'
if g2URL in g2repo.remote().urls:
return 'github'+rev
G2_installed_result = 'git'+rev
return G2_installed_result
except:
pass
if svnGetRev(verbose=False):
G2_installed_result = 'svn'
return G2_installed_result
G2_installed_result = 'noVCS'
return G2_installed_result
def GetVersionNumber():
'''Obtain a version number for GSAS-II from git, svn or from the
files themselves, if no other choice.
This routine was used to get the GSAS-II version from strings
placed in files by svn with the version number being the latest
number found, gathered by :func:`SetVersionNumber` (not 100% accurate
as the latest version might have files changed that are not tagged
by svn or with a call to SetVersionNumber. Post-svn this info
will not be reliable, and this mechanism is replaced by a something
created with a git hook, file git_verinfo.py (see the git_filters.py file).
Before resorting to the approaches above, try to ask git or svn
directly.
:returns: an int value usually, but a value of 'unknown' might occur
'''
if HowIsG2Installed().startswith('git'):
g2repo = openGitRepo(path2GSAS2)
for h in list(g2repo.iter_commits('HEAD'))[:50]: # (don't go too far back)
tags = g2repo.git.tag('--points-at',h).split('\n')
try:
for item in tags:
if item.isnumeric(): return int(item)
except:
pass
elif HowIsG2Installed() == 'svn':
rev = svnGetRev()
if rev is not None: return rev
# No luck asking, look up version information from git_verinfo.py
try:
import git_verinfo as gv
try:
for item in gv.git_tags:
if item.isnumeric(): return int(item)
except:
pass
try:
for item in gv.git_prevtags:
if item.isnumeric(): return int(item)
except:
pass
except:
pass
# all else failed, use the SetVersionNumber value
if version > 5000: # a small number must be wrong
return version
else:
return "unknown"
def getG2VersionInfo():
if HowIsG2Installed().startswith('git'):
g2repo = openGitRepo(path2GSAS2)
commit = g2repo.head.commit
ctim = commit.committed_datetime.strftime('%d-%b-%Y %H:%M')
now = dt.datetime.now().replace(
tzinfo=commit.committed_datetime.tzinfo)
delta = now - commit.committed_datetime
age = delta.total_seconds()/(60*60*24.)
tags = g2repo.git.tag('--points-at',commit).split('\n')
tags = [i for i in tags if i.isnumeric()]
gversion = "of "
if len(tags) >= 1:
gversion = f"#{tags[0]} "
msg = ''
if g2repo.head.is_detached:
msg = ("\n" +
"**** You have reverted to a past version of GSAS-II. Please \n"
+
"contact the developers with what is preferred in this version ****"
)
else:
rc,lc,_ = gitCheckForUpdates(False,g2repo)
if rc is None:
msg = "\nOn locally defined branch?"
elif age > 60 and len(rc) > 0:
msg = f"\n**** This version is really old. Please update. >= {len(rc)} updates have been posted ****"
elif age > 5 and len(rc) > 0:
msg = f"\n**** Please consider updating. >= {len(rc)} updates have been posted"
elif len(rc) > 0:
msg = f"\nThis GSAS-II version is ~{len(rc)} updates behind current."
return f"GSAS-II version {gversion} {ctim} ({age:.1f} days old). Git: {commit.hexsha[:6]}{msg}"
elif HowIsG2Installed() == 'svn':
rev = svnGetRev()
if rev is None:
"no SVN"
else:
rev = f"SVN version {rev}"
# patch 11/2020: warn if GSASII path has not been updated past v4576.
# For unknown reasons on Mac with gsas2full, there have been checksum
# errors in the .so files that prevented svn from completing updates.
# If GSASIIpath.svnChecksumPatch is not present, then the fix for that
# has not been retrieved, so warn. Keep for a year or so.
try:
svnChecksumPatch
except:
print('Warning GSAS-II incompletely updated. Please contact [email protected]')
# end patch
return f"Latest GSAS-II revision: {GetVersionNumber()} (svn {rev})"
else:
try:
import git_verinfo as gv
if gv.git_tags:
msg = f"{' '.join(gv.git_tags)}, Git: {gv.git_version[:6]}"
else:
msg = (f"{gv.git_version[:6]}; "+
f"Prev ver: {' '.join(gv.git_prevtags)}"+
f", {gv.git_prevtaggedversion[:6]}")
return f"GSAS-II version: {msg} (Manual update)"
except:
pass
# all else fails, use the old version number routine
return f"GSAS-II installed manually, last revision: {GetVersionNumber()}"
#==============================================================================
#==============================================================================
# routines to interface with git.
# next lines define where GitHub repositories are found
#g2URL = "https://github.com/AdvancedPhotonSource/GSASII-copy.git"
g2URL = "https://github.com/GSASII/codetest.git"
G2binURL = "https://api.github.com/repos/GSASII/binarytest"
gitOwner,gitRepo = 'GSASII', 'TutorialTest'
BASE_HEADER = {'Accept': 'application/vnd.github+json',
'X-GitHub-Api-Version': '2022-11-28'}
def openGitRepo(repo_path):
try: # patch 3/2024 for svn dir organization
return git.Repo(path2GSAS2)
except git.InvalidGitRepositoryError:
pass
return git.Repo(os.path.dirname(path2GSAS2))
def gitLookup(repo_path,gittag=None,githash=None):
'''Return information on a particular checked-in version
of GSAS-II.
:param str repo_path: location where GSAS-II has been installed
:param str gittag: a tag value.
:param str githash: hex hash code (abbreviated to as few characters as
needed to keep it unique). If None (default), a tag must be supplied.
:returns: either None if the tag/hash is not found or a tuple with
four values (hash, tag-list, message,date_time) where
* hash (str) is the git checking hash code;
* tag-list is a list of tags (typically there will
be one or two);
* message is the check-in message (str)
* date_time is the check-in date as a datetime object
'''
g2repo = openGitRepo(repo_path)
if gittag is not None and githash is not None:
raise ValueError("Cannot specify a hash and a tag")
if gittag is not None:
try:
commit = g2repo.tag(gittag).commit
except ValueError:
return None
elif githash is not None:
try:
commit = g2repo.commit(githash)
except git.BadName:
return None
else:
raise ValueError("Must specify either a hash or a tag")
tags = [i.name for i in g2repo.tags if i.commit == commit]
return (commit.hexsha, tags, commit.message,commit.committed_datetime)
def gitHash2Tags(githash=None,g2repo=None):
'''Find tags associated with a particular git commit.
Note that if `githash` cannot be located because it does not
exist or is not unique, a `git.BadName` exception is raised.
:param str githash: hex hash code (abbreviated to as few characters as
needed to keep it unique). If None (default), the HEAD is used.
:param str g2repo: git.Rwpo connecton to GSAS-II installation. If
None (default) it will be opened.
:returns: a list of tags (each a string)
'''
if g2repo is None:
g2repo = openGitRepo(path2GSAS2)
if githash is None:
commit = g2repo.head.object
else:
commit = g2repo.commit(githash)
#return [i.name for i in g2repo.tags if i.commit == commit] # slow with a big repo
return g2repo.git.tag('--points-at',commit).split('\n')
def gitTag2Hash(gittag,g2repo=None):
'''Provides the hash number for a git tag.
Note that if `gittag` cannot be located because it does not
exist or is too old and is beyond the `depth` of the local
repository, a `ValueError` exception is raised.
:param str repo_path: location where GSAS-II has been installed.
:param str gittag: a tag value.
:param str g2repo: git.Rwpo connecton to GSAS-II installation. If
None (default) it will be opened.
:returns: a str value with the hex hash for the commit.
'''
if g2repo is None:
g2repo = openGitRepo(path2GSAS2)
return g2repo.tag(gittag).commit.hexsha
def gitTestGSASII(verbose=True,g2repo=None):
'''Test a the status of a GSAS-II installation
:param bool verbose: if True (default), status messages are printed
:param str g2repo: git.Rwpo connecton to GSAS-II installation. If
None (default) it will be opened.
:returns: istat, with the status of the repository, with one of the
following values:
* -1: path is not found
* -2: no git repository at path
* -3: unable to access repository
* value&1==1: repository has local changes (uncommitted/stashed)
* value&2==2: repository has been regressed (detached head)
* value&4==4: repository has staged files
* value&8==8: repository has has been switched to non-master branch
* value==0: no problems noted
'''
if g2repo is None:
if not os.path.exists(path2GSAS2):
if verbose: print(f'Warning: Directory {path2GSAS2} not found')
return -1
if not os.path.exists(os.path.join(path2GSAS2,'.git')):
if verbose: print(f'Warning: Repository {path2GSAS2} not found')
return -2
try:
g2repo = openGitRepo(path2GSAS2)
except Exception as msg:
if verbose: print(f'Warning: Failed to open repository. Error: {msg}')
return -3
code = 0
if g2repo.is_dirty(): # has changed files
code += 1
count_modified_files = len(g2repo.index.diff(None))
if g2repo.head.is_detached:
code += 2 # detached
else:
if g2repo.active_branch.name != 'master':
code += 8 # not on master branch
if g2repo.index.diff("HEAD"): code += 4 # staged
# test if there are local changes committed
return code
def gitCheckForUpdates(fetch=True,g2repo=None):
'''Provides a list of the commits made locally and those in the
local copy of the repo that have not been applied. Does not
provide useful information in the case of a detached Head (see
:func:`countDetachedCommits` for that.)
:param bool fetch: if True (default), updates are copied over from
the remote repository (git fetch), before checking for changes.
:param str g2repo: git.Rwpo connecton to GSAS-II installation. If
None (default) it will be opened.
:returns: a list containing (remotecommits, localcommits, fetched) where
* remotecommits is a list of hex hash numbers of remote commits and
* localcommits is a list of hex hash numbers of local commits and
* fetched is a bool that will be True if the update (fetch)
step ran successfully
Note that if the head is detached (GSAS-II has been reverted to an
older version) or the branch has been changed, the values for each
of the three items above will be None.
'''
fetched = False
if g2repo is None:
g2repo = openGitRepo(path2GSAS2)
if g2repo.head.is_detached:
return (None,None,None)
if fetch:
try:
g2repo.remote().fetch()
fetched = True
except git.GitCommandError as msg:
print(f'Failed to get updates from {g2repo.remote().url}')
try:
head = g2repo.head.ref
tracking = head.tracking_branch()
localcommits = [i.hexsha for i in head.commit.iter_items(g2repo, f'{tracking.path}..{head.path}')]
remotecommits = [i.hexsha for i in head.commit.iter_items(g2repo, f'{head.path}..{tracking.path}')]
return remotecommits,localcommits,fetched
except:
return (None,None,None)
def countDetachedCommits(g2repo=None):
'''Count the number of commits that have been made since
a commit that is containined in the master branch
returns the count and the commit object for the
parent commit that connects the current stranded
branch to the master branch.
None is returned if no connection is found
'''
if g2repo is None:
g2repo = openGitRepo(path2GSAS2)
if not g2repo.head.is_detached:
return 0,g2repo.commit()
# is detached head in master branch?
if g2repo.commit() in g2repo.iter_commits('master'):
return 0,g2repo.commit()
# count number of commits since leaving master branch
masterList = list(g2repo.iter_commits('master'))
for c,i in enumerate(g2repo.commit().iter_parents()):
if i in masterList:
return c+1,i
else:
return None,None
def gitCountRegressions(g2repo=None):
'''Count the number of new check ins on the master branch since
the head was detached as well as any checkins made on the detached
head.
:returns: mastercount,detachedcount, where
* mastercount is the number of check ins made on the master branch
remote repository since the reverted check in was first made.
* detachedcount is the number of check ins made locally
starting from the detached head (hopefully 0)
If the connection between the current head and the master branch
cannot be established, None is returned for both.
If the connection from the reverted check in to the newest version
(I don't see how this could happen) then only mastercount will be None.
'''
if g2repo is None:
g2repo = openGitRepo(path2GSAS2)
# get parent of current head that is in master branch
detachedcount,parent = countDetachedCommits(g2repo)
if detachedcount is None: return None,None
mastercount = 0
for h in g2repo.iter_commits('master'):
if h == parent:
return mastercount,detachedcount
mastercount += 1
return None,detachedcount
def gitGetUpdate(mode='Background'):
'''Download the latest updates into the local copy of the GSAS-II
repository from the remote master, but don't actually update the
GSAS-II files being used. This can be done immediately or in background.
In 'Background' mode, a background process is launched. The results
from the process are recorded in file in ~/GSASII_bkgUpdate.log
(located in %HOME% on Windows). A pointer to the created process is
returned.
In 'immediate' mode, the update is performed immediately. The
function does not return until after the update is downloaded.
:returns: In 'Background' mode, returns a Popen object (see subprocess).
In 'immediate' mode nothing is returned.
'''
if mode == 'Background':
return subprocess.Popen([sys.executable, __file__, '--git-fetch'])
else:
g2repo = openGitRepo(path2GSAS2)
g2repo.remote().fetch()
if GetConfigValue('debug'): print(f'Updates fetched')
def gitHistory(values='tag',g2repo=None,maxdepth=100):
'''Provides the history of commits to the master, either as tags
or hash values
:param str values: specifies what type of values are returned.
If values=='hash', then hash values or for values=='tag', a
list of list of tag(s).
:param str g2repo: git.Rwpo connecton to GSAS-II installation. If
None (default) it will be opened.
:returns: a list of str values where each value is a hash for
a commit (values=='hash'),
for values=='tag', a list of lists, where a list of tags is provided
for each commit. When tags are provided, for any commit that does
not have any associated tag(s), that entry is omitted from the list.
for values=='both', a list of lists, where a hash is followed by a
list of tags (if any) is provided
'''
if g2repo is None:
g2repo = openGitRepo(path2GSAS2)
history = list(g2repo.iter_commits('master'))
if values.lower().startswith('h'):
return [i.hexsha for i in history]
elif values.lower().startswith('t'):
tagmap = {} # generate lookup table for to get tags
for t in g2repo.tags:
tagmap.setdefault(t.commit.hexsha, []).append(t.name)
return [tagmap[i.hexsha] for i in history if i.hexsha in tagmap]
elif values.lower().startswith('b'):
# slow with history >thousands
# tagmap = {} # generate lookup table for to get tags
# for t in g2repo.tags:
# tagmap.setdefault(t.commit.hexsha, []).append(t.name)
# return [[i.hexsha]+tagmap.get(i.hexsha,[]) for i in history]
# potentially faster code
r1 = [[i.hexsha]+g2repo.git.tag('--points-at',i).split('\n')
for i in history[:maxdepth]]
return [[i[0]] if i[1]=='' else i for i in r1]
else:
raise ValueError(f'gitHistory has invalid value specified: {value}')
def getGitBinaryReleases():
'''Retrieves the binaries and download urls of the latest release
:returns: a URL dict for GSAS-II binary distributions found in the newest
release in a GitHub repository. The repo location is defined in global
`G2binURL`.
The dict keys are references to binary distributions, which are named
as f"{platform}_p{pver}_n{npver}" where platform is determined
in :func:`GSASIIpath.GetBinaryPrefix` (linux_64, mac_arm, win_64,...)
and where `pver` is the Python version (such as "3.10") and `npver` is
the numpy version (such as "1.26").
The value associated with each key contains the full URL to
download a tar containing that binary distribution.
'''
# Get first page of releases
releases = requests.get(
url=f"{G2binURL}/releases",
headers=BASE_HEADER
).json()
# Get assets of latest release
assets = requests.get(
url=f"{G2binURL}/releases/{releases[-1]['id']}/assets",
headers=BASE_HEADER
).json()
versions = []
URLs = []
for asset in assets:
if asset['name'].endswith('.tgz'):
versions.append(asset['name'][:-4]) # Remove .tgz tail
URLs.append(asset['browser_download_url'])
return dict(zip(versions,URLs))
def getGitBinaryLoc(npver=None,pyver=None,verbose=True):
'''Identify the best GSAS-II binary download location from the
distributions in the latest release section of the github repository
on the CPU platform, and Python & numpy versions. The CPU & Python
versions must match, but the numpy version may only be close.
:param str npver: Version number to use for numpy, if None (default)
the version is taken from numpy in the current Python interpreter.
:param str pyver: Version number to use for Python, if None (default)
the version is taken from the current Python interpreter.
:param bool verbose: if True (default), status messages are printed
:returns: a URL for the tar file (success) or None (failure)
'''
bindir = GetBinaryPrefix(pyver)
if npver:
inpver = intver(npver)
else:
npver = np.__version__
inpver = intver(np.__version__)
# get binaries matching the required install, approximate match for numpy
URLdict = getGitBinaryReleases()
versions = {}
for d in URLdict:
if d.startswith(bindir):
v = intver(d.rstrip('/').split('_')[3].lstrip('n'))
versions[v] = d
intVersionsList = sorted(versions.keys())
if not intVersionsList:
print('No binaries located to match',bindir)
return
elif inpver < min(intVersionsList):
vsel = min(intVersionsList)
if verbose: print(
f'Warning: The requested numpy, version, {npver},'
f' is older than\n\tthe oldest dist version, {fmtver(vsel)}')
elif inpver >= max(intVersionsList):
vsel = max(intVersionsList)
if verbose and inpver == max(intVersionsList):
print(
f'The requested numpy version, {npver},'
f' matches the binary dist, version {fmtver(vsel)}')
elif verbose:
print(
f'Note: using a binary dist for numpy version {fmtver(vsel)} '
f'which is older than the requested numpy, version {npver}')
else:
vsel = min(intVersionsList)
for v in intVersionsList:
if v <= inpver:
vsel = v
else:
if verbose: print(
f'FYI: Selecting dist version {fmtver(vsel)}'
f' as the requested numpy, version, {npver},'
f'\n\tis older than the next dist version {fmtver(v)}')
break
return URLdict[versions[vsel]]
def InstallGitBinary(tarURL, instDir, nameByVersion=False, verbose=True):
'''Install the GSAS-II binary files into the location
specified.
:param str tarURL: a URL for the tar file.
:param str instDir: location directory to install files. This directory
may not exist and will be created if needed.
:param bool nameByVersion: if True, files are put into a subdirectory
of `instDir`, named to match the tar file (with plaform, Python &
numpy versions).
Default is False, where the binary files are put directly into
`instDir`.
:param bool verbose: if True (default), status messages are printed.
:returns: None
'''
# packages not commonly used so import them here not on startup
import requests
import tempfile
import tarfile
# download to scratch
tar = tempfile.NamedTemporaryFile(suffix='.tgz',delete=False)
try:
tar.close()
if verbose: print(f'Downloading {tarURL}')
r = requests.get(tarURL, allow_redirects=True)
open(tar.name, 'wb').write(r.content)
# open in tar
tarobj = tarfile.open(name=tar.name)
if nameByVersion:
binnam = os.path.splitext(os.path.split(tarURL)[1])[0]
install2dir = os.path.join(instDir,binnam)
else:
install2dir = instDir
for f in tarobj.getmembers(): # loop over files in repository
# do a bit of sanity checking for safety. Don't install anything
# unless it goes into in the specified directory
if '/' in f.name or '\\' in f.name:
print(f'skipping file {f.name} -- path alteration not allowed')
continue
if f.name != os.path.basename(f.name):
print(f'skipping file {f.name} -- how did this happen?')
continue
newfil = os.path.normpath(os.path.join(install2dir,f.name))
tarobj.extract(f, path=install2dir, set_attrs=False)
# set file mode and mod/access times (but not ownership)
os.chmod(newfil,f.mode)
os.utime(newfil,(f.mtime,f.mtime))
if verbose: print(f'Created GSAS-II binary file {newfil}')
finally:
del tarobj
os.unlink(tar.name)
def GetRepoUpdatesInBackground():
'''Wrapper to make sure that :func:`gitGetUpdate` is called only
if git has been used to install GSAS-II.
:returns: returns a Popen object (see subprocess)
'''
if HowIsG2Installed().startswith('git'):
return gitGetUpdate(mode='Background')
def gitStartUpdate(cmdopts):
'''Update GSAS-II in a separate process, by running this script with the
options supplied in the call to this function and then exiting GSAS-II.
'''
cmd = [sys.executable, __file__] + cmdopts
if GetConfigValue('debug'): print('Starting updates with command\n\t'+
f'{" ".join(cmd)}')
proc = subprocess.Popen(cmd)
# on windows the current process needs to end so that the source files can
# be written over. On unix the current process needs to stay running
# so the child is not killed.
if sys.platform != "win32": proc.wait()
sys.exit()
def dirGitHub(dirlist,orgName=gitOwner, repoName=gitRepo):
'''Obtain a the contents of a GitHub repository directory using
the GitHub REST API.
:param str dirlist: a list of sub-directories `['parent','child',sub']`
for `parent/child/sub` or `[]` for a file in the top-level directory.
:param str orgName: the name of the GitHub organization
:param str repoName: the name of the GitHub repository
:returns: a list of file names or None if the dirlist info does not
reference a directory
examples::
dirGitHub([], 'GSASII', 'TutorialTest')
dirGitHub(['TOF Sequential Single Peak Fit', 'data'])
The first example will get the contents of the top-level
directory for the specified repository
The second example will provide the contents of the
"TOF Sequential Single Peak Fit"/data directory.
'''
dirname = ''
for item in dirlist:
dirname += item + '/'
URL = f"https://api.github.com/repos/{orgName}/{repoName}/contents/{dirname}"
r = requests.get(URL, allow_redirects=True)
try:
return [rec['name'] for rec in r.json()]
except:
return None
def rawGitHubURL(dirlist,filename,orgName=gitOwner, repoName=gitRepo,
branchname="master"):
'''Create a link that can be used to view/downlaod the raw version of
file in a GitHub repository.
:param str dirlist: a list of sub-directories `['parent','child',sub']`
for `parent/child/sub` or `[]` for a file in the top-level directory.
:param str filename: the name of the file
:param str orgName: the name of the GitHub organization
:param str repoName: the name of the GitHub repository
:param str branchname: the name of the GitHub branch. Defaults
to "master".
:returns: a URL-encoded URL
'''
import urllib.parse # not used very often, import only when needed
dirname = ''
for item in dirlist:
# it's not clear that the URLencode is needed for the directory name
dirname += urllib.parse.quote(item) + '/'
#filename = urllib.parse.quote(filename)
return f"https://raw.githubusercontent.com/{orgName}/{repoName}/{branchname}/{dirname}{filename}"
def downloadDirContents(dirlist,targetDir,orgName=gitOwner, repoName=gitRepo):
filList = dirGitHub(dirlist, orgName=orgName, repoName=repoName)
if filList is None:
print(f'Directory {"/".join(dirlist)!r} does not have any files')
return None
for fil in filList:
if fil.lower() == 'index.html': continue
URL = rawGitHubURL(dirlist,fil,orgName=orgName,repoName=repoName)
r = requests.get(URL, allow_redirects=True)
outfil = os.path.join(targetDir,fil)
if r.status_code == 200:
open(outfil, 'wb').write(r.content)
print(f'wrote {outfil}')
elif r.status_code == 404:
print(f'Warning: {fil} is likley a subdirectory of directory {"/".join(dirlist)!r}')
else:
print(f'Unexpected web response for {fil}: {r.status_code}')
return
#==============================================================================
#==============================================================================
# routines to interface with subversion
g2home = 'https://subversion.xray.aps.anl.gov/pyGSAS' # 'Define the location of the GSAS-II subversion repository'
proxycmds = [] # 'Used to hold proxy information for subversion, set if needed in whichsvn'
svnLocCache = None # 'Cached location of svn to avoid multiple searches for it'
def MakeByte2str(arg):
'''Convert output from subprocess pipes (bytes) to str (unicode) in Python 3.
In Python 2: Leaves output alone (already str).
Leaves stuff of other types alone (including unicode in Py2)
Works recursively for string-like stuff in nested loops and tuples.
typical use::
out = MakeByte2str(out)
or::
out,err = MakeByte2str(s.communicate())
'''
if isinstance(arg,str): return arg
if isinstance(arg,bytes):
try:
return arg.decode()
except:
if GetConfigValue('debug'): print('Decode error')
return arg
if isinstance(arg,list):
return [MakeByte2str(i) for i in arg]
if isinstance(arg,tuple):
return tuple([MakeByte2str(i) for i in arg])
return arg
def getsvnProxy():
'''Loads a proxy for subversion from the proxyinfo.txt file created
by bootstrap.py or File => Edit Proxy...; If not found, then the
standard http_proxy and https_proxy environment variables are scanned
(see https://docs.python.org/3/library/urllib.request.html#urllib.request.getproxies)
with case ignored and that is used.
'''
global proxycmds
proxycmds = []
proxyinfo = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
if not os.path.exists(proxyinfo):
proxyinfo = os.path.join(path2GSAS2,"proxyinfo.txt")
if os.path.exists(proxyinfo):
fp = open(proxyinfo,'r')
host = fp.readline().strip()
# allow file to begin with comments
while host.startswith('#'):
host = fp.readline().strip()
port = fp.readline().strip()
etc = []
line = fp.readline()
while line:
etc.append(line.strip())
line = fp.readline()
fp.close()
setsvnProxy(host,port,etc)
return host,port,etc
import urllib.request
proxdict = urllib.request.getproxies()
varlist = ("https","http")
for var in proxdict:
if var.lower() in varlist:
proxy = proxdict[var]
pl = proxy.split(':')
if len(pl) < 2: continue
host = pl[1].strip('/')
port = ''
if len(pl) == 3:
port = pl[2].strip('/').strip()
return host,port,''
return '','',''
def setsvnProxy(host,port,etc=[]):
'''Sets the svn commands needed to use a proxy
'''
global proxycmds
proxycmds = []
host = host.strip()
port = port.strip()
if host:
proxycmds.append('--config-option')
proxycmds.append('servers:global:http-proxy-host='+host)
if port:
proxycmds.append('--config-option')
proxycmds.append('servers:global:http-proxy-port='+port)
for item in etc:
proxycmds.append(item)
def whichsvn():
'''Returns a path to the subversion exe file, if any is found.
Searches the current path after adding likely places where GSAS-II
might install svn.
:returns: None if svn is not found or an absolute path to the subversion
executable file.
'''
# use a previosuly cached svn location
global svnLocCache
if svnLocCache: return svnLocCache
# prepare to find svn
is_exe = lambda fpath: os.path.isfile(fpath) and os.access(fpath, os.X_OK)
svnprog = 'svn'
if sys.platform.startswith('win'): svnprog += '.exe'
host,port,etc = getsvnProxy()
if GetConfigValue('debug') and host:
print('DBG_Using proxy host {} port {}'.format(host,port))
if GetConfigValue('svn_exec'):
exe_file = GetConfigValue('svn_exec')
print('Using ',exe_file)
if is_exe(exe_file):
try:
p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
res = p.stdout.read()
if not res: return
p.communicate()
svnLocCache = os.path.abspath(exe_file)
return svnLocCache
except:
pass
# add likely places to find subversion when installed with GSAS-II
pathlist = os.environ["PATH"].split(os.pathsep)
pathlist.insert(0,os.path.split(sys.executable)[0])
pathlist.insert(1,path2GSAS2)