Merge branch 'webobs-2.7'

Author: beaudu

.gitignore

@@ -16,5 +16,7 @@ PACK/*
 !PACK/fpm*
 !PACK/showi18id
 !PACK/CHECKLIST.txt
+*.swo
+*.swp
 .DS_Store
 **/.DS_Store

CODE/cgi-bin/formEXTENSO.pl

@@ -51,7 +51,7 @@ =head1 Query string parameter
 
 die "You can't edit EXTENSO reports." if (!clientHasEdit(type=>"authforms",name=>"EXTENSO"));
 
-my $FORM = new WebObs::Form('EXTENSO_LEG');
+my $FORM = new WebObs::Form('EXTENSO');
 my %Ns;
 my @NODESSelList;
 my %Ps = $FORM->procs;

CODE/cgi-bin/mailInfo_OVPF.pl

@@ -81,6 +81,8 @@ =head1 Query string parameters
 # ---- standard FORMS inits ----------------------------------
 my $QP       = $cgi->Vars;
 
+$QP->{'form'}    ||= "EXTENSO";
+
 die "You can't view EXTENSO reports." if (!clientHasRead(type=>"authforms",name=>$QP->{form}));
 my $displayOnly = clientHasEdit(type=>"authforms",name=>$QP->{form}) ? 0 : 1;
 

CODE/cgi-bin/mailInfo_REVOSIMA.pl

@@ -21,6 +21,8 @@ winston|EARTHWORM|EarthWorm Winston Wave Server data request|
 gipsy|GNSS|JPL GIPSY-OASIS .tdp file|ANTENNA,RECEIVER,XYZ,CHAR9
 gipsyx|GNSS|JPL GipsyX .tdp file|ANTENNA,RECEIVER,CHAR9
 globkval|GNSS|MIT GAMIT/GLOBK VAL file|CHAR9
+spotgins-ippp|GNSS|SPOTGINS IPPP timeseries|CHAR9
+gamit-pos|GNSS|MIT GAMIT/GLOBL POS timeseries|CHAR9
 sbe37-ascii|GNSS|sbe37 OBP text file|DATA_ERROR,DATA_DECIMATE,CHAR9
 usgs-rneu|GNSS|USGS RNEU text file|CHAR9
 ics-neu|GNSS|ICS NEU text file|CHAR9

CODE/cgi-bin/showEXTENSO.pl

@@ -24,7 +24,7 @@
 %
 %	Authors: François Beauducel, Jean-Marie Saurel, WEBOBS/IPGP
 %	Created: 2013-12-29, in Guadeloupe, French West Indies
-%	Updated: 2024-07-02
+%	Updated: 2025-01-23
 
 wofun = sprintf('WEBOBS{%s}',mfilename);
 
@@ -72,7 +72,7 @@
 		case {'miniseed','seedlink','arclink','combined','fdsnws-dataselect'}
 			D(n) = readfmtdata_miniseed(WO,P,N(n),F);
 
-		case {'globkval','gipsy','gipsyx','gipsy-tdp','usgs-rneu','ies-neu','ogc-neu','ingv-gps','sbe37-ascii'}
+		case {'globkval','gipsy','gipsyx','gipsy-tdp','usgs-rneu','ies-neu','ogc-neu','ingv-gps','sbe37-ascii','spotgins-ippp','gamit-pos'}
 			D(n) = readfmtdata_gnss(WO,P,N(n),F);
 
 		case {'hyp71sum2k','fdsnws-event','scevtlog-xml'}

CODE/etc/rawformats.conf

@@ -34,6 +34,18 @@
 %		data format: extract from SmoothFinal.tdp output file (grep "Station.SSSS.State.pos.[XYZ]" lines)
 %		node calibration: no .CLB file or 4 components (East, North, Up) in meters and (Orbit)
 %
+%	format 'gins-ippp'
+%		type: GINS IPPP solutions
+%		filename/url: P.RAWDATA (use $FID to point the right file/url)
+%		data format: yyyymmdd hhmmss yyyy.yyyyyyyyy jjjjj.jj X Y Z dX dY dZ E N V dE dN dV
+%		node calibration: no .CLB file or 4 components (East, North, Up) in meters and (Orbit)
+%
+%	format 'gamit-pos'
+%		type: GAMIT/GLOBK POS file
+%		filename/url: P.RAWDATA (use $FID to point the right file/url)
+%		data format: yyyymmdd jjjjj.jjjj E N U dE dN dU
+%		node calibration: no .CLB file or 4 components (East, North, Up) in meters and (Orbit)
+%
 %	format 'usgs-rneu'
 %		type: USGS GPS results ITRF08
 %		filename/url: P.RAWDATA (use $FID to point the right file/url)
@@ -55,10 +67,12 @@
 %
 %	Authors: François Beauducel and Jean-Bernard de Chabalier, WEBOBS/IPGP
 %	Created: 2016-07-10, in Yogyakarta (Indonesia)
-%	Updated: 2021-11-12
+%	Updated: 2025-01-23
 
 wofun = sprintf('WEBOBS{%s}',mfilename);
 
+% minimum decent error is 0.1 mm (!)
+min_error = 1e-4;
 
 switch F.fmt
 % -----------------------------------------------------------------------------
@@ -118,6 +132,50 @@
 	%end
 	%D.ITRF_YEAR = itrf;
 
+
+% -----------------------------------------------------------------------------
+case 'gamit-pos'
+	% format example
+	% yyyymmdd jjjjj.jjjj  E       N        U       dE      dN      dU
+    % 20100101 55197.4993 -0.09875 -0.11953 0.00075 0.00198 0.00161 0.00645
+
+	fdat = sprintf('%s/%s.dat',F.ptmp,N.ID);
+	wosystem(sprintf('rm -f %s',fdat),P);
+	for a = 1:length(F.raw)
+		fraw = F.raw{a};
+		cmd0 = sprintf('awk ''/^[^!]/ {print}'' >> %s',fdat); % removes header lines
+		if strncmpi('http',fraw,4)
+			s  = wosystem(sprintf('curl -s -S "%s" | %s',fraw,cmd0),P);
+			if s ~= 0
+				break;
+			end
+		else
+			s = wosystem(sprintf('cat %s | %s',fraw,cmd0),P);
+		end
+		if s ~= 0
+			fprintf('%s: ** WARNING ** Raw data "%s" not found.\n',wofun,fraw);
+		end
+	end
+
+	% load the file
+	if exist(fdat,'file')
+		dd = load(fdat);
+	else
+		dd = [];
+	end
+	if ~isempty(dd)
+		t = dd(:,2) + 678941.5007; % converts MJD to datenum
+		d = [dd(:,3:5),zeros(size(dd,1),1)];	% North(mm),East(mm),Up(mm) => E(m),N(m),U(m),Orbit
+		e = dd(:,6:8);
+		e(e<min_error) = min_error;
+		fprintf('%d data imported.\n',size(dd,1));
+	else
+		fprintf('no data found!\n')
+		t = [];
+		d = [];
+		e = [];
+	end
+
 % -----------------------------------------------------------------------------
 case {'gipsy','gipsy-tdp','gipsyx'}
 
@@ -183,8 +241,7 @@
 		% converts cartesian geocentric (X,Y,Z) to UTM, estimating errors
 		[enu,e] = cart2utm(dd(:,[2,6,10])*kmfact,dd(:,[3,7,11])*kmfact);
 		d = [enu,dd(:,4)];
-		% minimum decent error is 1 mm (!)
-		e(e<1e-3) = 1e-3;
+		e(e<min_error) = min_error;
 		fprintf(' %d data imported.\n',size(dd,1));
 	else
 		fprintf(' no data found!\n')
@@ -194,6 +251,49 @@
 	end
 	%D.ITRF_YEAR = 'ITRF08';
 
+% -----------------------------------------------------------------------------
+case 'spotgins-ippp'
+	% format example
+	% !yyyymmdd hhmmss yyyy.yyyyyyyyy  jjjjj.jj        X_position        Y_position        Z_position            dX            dY            dZ             E             N             V            dE            dN            dV
+    %  20160723  65619 2016.558521561  57592.29    4182067.152057     570976.439258    4765940.539811      0.000611      0.000218      0.000673     -0.006574     -0.008848     -0.014844      0.000205      0.000307      0.000859
+
+	fdat = sprintf('%s/%s.dat',F.ptmp,N.ID);
+	wosystem(sprintf('rm -f %s',fdat),P);
+	for a = 1:length(F.raw)
+		fraw = F.raw{a};
+		cmd0 = sprintf('awk ''/^[^!]/ {print}'' >> %s',fdat); % removes header lines
+		if strncmpi('http',fraw,4)
+			s  = wosystem(sprintf('curl -s -S "%s" | %s',fraw,cmd0),P);
+			if s ~= 0
+				break;
+			end
+		else
+			s = wosystem(sprintf('cat %s | %s',fraw,cmd0),P);
+		end
+		if s ~= 0
+			fprintf('%s: ** WARNING ** Raw data "%s" not found.\n',wofun,fraw);
+		end
+	end
+
+	% load the file
+	if exist(fdat,'file')
+		dd = load(fdat);
+	else
+		dd = [];
+	end
+	if ~isempty(dd)
+		t = datenum(dd(:,3),1,1,0,0,0);
+		d = [dd(:,11:13),zeros(size(dd,1),1)];	% North(mm),East(mm),Up(mm) => E(m),N(m),U(m),Orbit
+		e = dd(:,14:16);
+		e(e<min_error) = min_error;
+		fprintf('%d data imported.\n',size(dd,1));
+	else
+		fprintf('no data found!\n')
+		t = [];
+		d = [];
+		e = [];
+	end
+
 % -----------------------------------------------------------------------------
 case 'usgs-rneu'
 	% format example
@@ -230,8 +330,7 @@
 		t = datenum(ty,tm,td,12,0,0);	% date is YYYYMMDD and we force time to 12:00:00
 		d = [dd(:,[3,2,4])/1e3,dd(:,5)];	% North(mm),East(mm),Up(mm),Orbit => E(m),N(m),U(m),O
 		e = dd(:,[7,6,8])/1e3;
-		% minimum decent error is 1 mm (!)
-		e(e<1e-3) = 1e-3;
+		e(e<min_error) = min_error;
 		fprintf('%d data imported.\n',size(dd,1));
 	else
 		fprintf('no data found!\n')
@@ -274,8 +373,7 @@
 		t = datenum(dd(:,1),1,1,0,0,0);	% date is decimal year
 		d = [dd(:,[4,2,6]),zeros(size(dd,1),1)];	% North(mm),East(mm),Up(mm),Orbit => E(m),N(m),U(m),O
 		e = dd(:,[5,3,7]);
-		% minimum decent error is 1 mm (!)
-		e(e<1e-3) = 1e-3;
+		e(e<min_error) = min_error;
 		fprintf('%d data imported.\n',size(dd,1));
 	else
 		fprintf('no data found!\n')
@@ -318,8 +416,7 @@
 		t = datenum(dd(:,2),dd(:,3),dd(:,4),12,0,0);	% here we force time to noon
 		d = [dd(:,[6,5,7]),zeros(size(dd,1),1)];	% North(mm),East(mm),Up(mm) => E(m),N(m),U(m),Orbit
 		e = dd(:,[9,8,10]);
-		% minimum decent error is 1 mm (!)
-		e(e<1e-3) = 1e-3;
+		e(e<min_error) = min_error;
 		fprintf('%d data imported.\n',size(dd,1));
 	else
 		fprintf('no data found!\n')
@@ -367,8 +464,7 @@
 		t = datenum(dd(:,1),1,1);	% here we force time to January 1st!
 		d = [dd(:,2:4),zeros(size(dd,1),1)];
 		e = dd(:,5:7);
-		% minimum decent error is 1 mm (!)
-		e(e<1e-3) = 1e-3;
+		e(e<min_error) = min_error;
 		fprintf('%d data imported.\n',size(dd,1));
 	else
 		fprintf('no data found!\n')

CODE/matlab/readfmtdata.m

@@ -40,7 +40,7 @@ function gridmaps(grids,outd,varargin)
 %
 %   Author: F. Beauducel, C. Brunet, WEBOBS/IPGP
 %   Created: 2013-09-13 in Paris, France
-%   Updated: 2024-12-04
+%   Updated: 2025-01-23
 
 
 WO = readcfg;
@@ -211,11 +211,11 @@ function gridmaps(grids,outd,varargin)
 
 	% looks for supplementary maps (MAP*_XYLIM|LON1,LON2,LAT1,LAT2 keys)
 	if merge
-		maps(1,:) = {'MAP',mmaplim};
+		maps = {'MAP',mmaplim};
 	else
 		fd = fieldnames(G);
 		k = find(~cellfun(@isempty,regexp(fd,'^MAP\d+_XYLIM$')));
-		maps(1,:) = {'MAP',[]};
+		maps = {'MAP',[]};
 		for ii = 1:length(k)
 			x = split(fd{k(ii)},'_');
 			v = G.(fd{k(ii)});

CODE/matlab/readfmtdata_gnss.m

@@ -1,7 +1,8 @@
 # WebObs - DEV : calibration file ISBFDF0
-# [2014-07-22 !]
-1998-11-25|00:00|1|LHE|nm/s|STS-2|LHE|0|3.9736e-1|1|||90|14.734971|-61.146311|467|0|1||00
-1998-11-25|00:00|2|LHN|nm/s|STS-2|LHN|0|3.9736e-1|1|||0|14.734971|-61.146311|467|0|1||00
-1998-11-25|00:00|3|LHZ|nm/s|STS-2|LHZ|0|3.9736e-1|1|||0|14.734971|-61.146311|467|0|1||00
-1998-11-25|00:00|4|LKI|�C|PT1000|LKI|20.78|2.841e-3|1|||0|14.734971|-61.146311|467|0|1||00
-1998-11-25|00:00|5|LDI|kPa|EFFA|LDI|90|0.6143e-3|1|||0|14.734971|-61.146311|467|0|1||00
+# [2020-02-05 !]
+=key|DATE|TIME|nv|nm|un|ns|cd|of|et|ga|vn|vm||||al|dp|sf|db|lc
+1|1998-11-25|00:00|1|LHE|m/s|STS-2|LHE|0|3.9736e-10|1|||90|14.734971|-61.146311|467|0|1||00
+2|1998-11-25|00:00|2|LHN|m/s|STS-2|LHN|0|3.9736e-10|1|||0|14.734971|-61.146311|467|0|1||00
+3|1998-11-25|00:00|3|LHZ|m/s|STS-2|LHZ|0|3.9736e-10|1|||0|14.734971|-61.146311|467|0|1||00
+4|1998-11-25|00:00|4|LKI|�C|PT1000|LKI|20.78|2.841e-3|1|||0|14.734971|-61.146311|467|0|1||00
+5|1998-11-25|00:00|5|LDI|Pa|EFFA|LDI|90000|0.6143|1|||0|14.734971|-61.146311|467|0|1||00

CODE/matlab/superprocs/gridmaps.m

@@ -1,7 +1,8 @@
-# WebObs - DEV : calibration file ISBFDFM
-# [2021-01-21 !]
-2019-10-24|00:00|1|LHE|nm/s|STS-2|LHE|0|3.9736e-1|1|||90|14.734971|-61.146311|467|0|1||00
-2019-10-24|00:00|2|LHN|nm/s|STS-2|LHN|0|3.9736e-1|1|||0|14.734971|-61.146311|467|0|1||00
-2019-10-24|00:00|3|LHZ|nm/s|STS-2|LHZ|0|3.9736e-1|1|||0|14.734971|-61.146311|467|0|1||00
-2019-10-24|00:00|4|LKO|�C|EFFA|LDI|0|0.6143|1|||0|14.734971|-61.146311|467|0|1||00
-2019-10-24|00:00|5|LDI|kPa|EFFA|LDI|90|0.6143e-3|1|||0|14.734971|-61.146311|467|0|1||00
+# WebObs - DEV-MARENGO : calibration file PROC.GEOSCOPE.ISBFDFM
+# [2024-04-26 !]
+=key|DATE|TIME|nv|nm|un|ns|cd|of|et|ga|vn|vm|az|la|lo|al|dp|sf|db|lc
+1|2019-10-24|00:00|1|LHE|nm/s|STS-2|LHE|0|3.9736e-1|1|||90|14.734971|-61.146311|467|0|1||00|
+2|2019-10-24|00:00|2|LHN|nm/s|STS-2|LHN|0|3.9736e-1|1|||0|14.734971|-61.146311|467|0|1||00|
+3|2019-10-24|00:00|3|LHZ|nm/s|STS-2|LHZ|0|3.9736e-1|1|||0|14.734971|-61.146311|467|0|1||00|
+4|2019-10-24|00:00|4|LKO|�C|EFFA|LDI|0|0.6143|1|||0|14.734971|-61.146311|467|0|1||00|
+5|2019-10-24|00:00|5|LDI|kPa|EFFA|LDI|90|0.6143e-3|1|||0|14.734971|-61.146311|467|0|1||00|

SETUP/DATA/NODES/GCSBCM1/GCSBCM1.clb

@@ -1,5 +1,6 @@
 # WEBOBS - DEV IPGP : calibration file WDCDHS0
 # [2013-04-03 FB]
-2008-06-21|00:00|1|Eastern|m||E|0|1|1|||0|16.27268|-61.76509|618|0|||
-2008-06-21|00:00|2|Northern|m||N|0|1|1|||0|16.27268|-61.76509|618|0|||
-2008-06-21|00:00|3|Vertical|m||U|0|1|1|||0|16.27268|-61.76509|618|0|||
+=key|DATE|TIME|nv|nm|un|ns|cd|of|et|ga|vn|vm|az|la|lo|al|dp|sf|db|lc
+1|2008-06-21|00:00|1|Eastern|m||E|0|1|1|||0|16.27268|-61.76509|618|0|||
+2|2008-06-21|00:00|2|Northern|m||N|0|1|1|||0|16.27268|-61.76509|618|0|||
+3|2008-06-21|00:00|3|Vertical|m||U|0|1|1|||0|16.27268|-61.76509|618|0|||

SETUP/DATA/NODES/GCSBJN1/GCSBJN1.clb

@@ -8,7 +8,7 @@ The latest release contains improvements, new features, bug fixes, and sometimes
 
 Sections with `!!` prefix must be carefully read in case of upgrade. It usually means that the upgrade could change some behavior from previous release installations (i.e., not a bug fix). An appropriate configuration to keep the former behavior is usually proposed.
 
-## v2.7.1 (December 2024)
+## v2.7.2 (January 2025)
 
 ### New features
 1. **Generic forms**: see section *Under development* below.
@@ -40,6 +40,7 @@ REPORT_FOOTNOTE|Si vous avez ressenti ce séisme, merci de témoigner sur le sit
 1. Better check of the consistency of node start/end dates when editing the configuration.
 1. Exported list of geolocated nodes as KML file has now extended data (start/end dates, active flag).
 1. Metadata can now be uploaded to the Theia|OZCAR data portal. A checkbox has been added to select and save channels to be uploaded.
+1. New GNSS solutions data formats: `spotgins-ippp`and `gamit-pos`.
 
 ### Fixed issues
 1. ``!!`` WebObs Perl modules have been moved from CODE/cgi-bin/WebObs to CODE/perl/lib. The system-wide installation **MUST** be executed when upgrading (answer Y to the appropriate question during setup) in order to make all CGI working.
@@ -54,9 +55,10 @@ REPORT_FOOTNOTE|Si vous avez ressenti ce séisme, merci de témoigner sur le sit
 1. Fix an issue with data files Campbell binary format TOB1 that contain no data but a header.
 1. Fix an issue in **gridmaps** with inactive nodes plotted over active ones.
 1. Fix a possible issue with **scevtlog-xml** data format (ignoring events created late after origin time).
+1. Fix forgotten binaries for Arclink rawformat.
 
 ### Under development
-1. **Generic forms**: it is now possible to create any user-defined manual database associated to a form (for entering new data and editing), a table display, and CSV export of data, using SQLite database. This new feature will replace all the actuel forms (EAUX, GAZ, EXTENSO, FISSURO, DISTANCE, NOVAC, SOILSOLUTIONS, and RAINWATER) in future release, with an automatic migration tool. Creating and using the new forms is functional, but reading the data from PROCS is not yet implemented. See the user manual documentation for details. 
+1. **Generic forms**: it is now possible to create any user-defined manual database associated to a form (for entering new data and editing), a table display, and CSV export of data, using SQLite database. This new feature will replace all the actuel forms (EAUX, GAZ, EXTENSO, FISSURO, DISTANCE, SOILSOLUTIONS, and RAINWATER) in future release, with an automatic migration tool. Creating and using the new forms is functional, but reading the data from PROCS is not yet implemented. See the user manual documentation for details. 
 
 ## v2.6.4 (December 2023)