Merge branch 'main' into python310

Author: rtburns-jpl

.circleci/config.yml

@@ -2,51 +2,36 @@ version: 2.1
 jobs:
   test-cmake:
     docker:
-        - image: hysds/pge-base:latest
-          user: root
+      - image: ubuntu:20.04
     steps:
       - checkout:
           path: /root/project/src
 
-      - run:
-          name: Install development tools
-          command: |
-            set -ex
-            pwd
-            yum update -y
-            yum groupinstall -y "development tools"
-
       - run:
           name: Install ISCE requirements
           command: |
             set -ex
             pwd
-            mkdir config build install
-            . /opt/conda/bin/activate root
-            conda install -y cython gdal h5py libgdal pytest numpy fftw scipy scons hdf4 hdf5 libgcc libstdcxx-ng cmake astropy pybind11
-            yum install -y x11-devel motif-devel jq gcc-gfortran opencv opencv-devel opencv-python
+            export DEBIAN_FRONTEND=noninteractive
+            apt-get update
+            apt-get install -y cmake cython3 git libfftw3-dev libgdal-dev libhdf4-alt-dev libhdf5-dev libopencv-dev python3-gdal python3-h5py python3-numpy python3-scipy
 
       - run:
           name: Build and Install ISCE
           command: |
             set -ex
             cd /root/project/src
-            . /opt/conda/bin/activate root
             mkdir build
             cd build
-            INSTALLPATH=/opt/conda
-            MODPATH=$(python3 -c "import site; print(site.getsitepackages()[0])")
-            # convert to relative path
-            MODPATH=$(realpath --relative-to=$INSTALLPATH $MODPATH)
-            cmake .. -DCMAKE_INSTALL_PREFIX=$INSTALLPATH -DPYTHON_MODULE_DIR=$MODPATH
+            MODPATH=$(python3 -c "import site; print(site.getsitepackages()[-1])")
+            cmake .. -DCMAKE_INSTALL_PREFIX=install -DPYTHON_MODULE_DIR=$MODPATH
             make install VERBOSE=y
 
       - run:
           name: Test ISCE installation
           command: |
             set -ex
             cd /root/project/src/build
-            . /opt/conda/bin/activate root
             ctest --output-on-failure
             ISCE2DIR=$(python3 -c "import os, isce2; print(os.path.dirname(isce2.__file__))" | tail -n 1)
             export PATH=$ISCE2DIR/applications:$PATH
@@ -60,37 +45,20 @@ jobs:
 
   test:
     docker:
-        - image: hysds/pge-base:latest
-          user: root
+      - image: ubuntu:20.04
     steps:
       - checkout:
           path: /root/project/src 
 
-      - run:
-          name: Install development tools
-          command: |
-            set -ex
-            pwd
-            yum update -y
-            yum groupinstall -y "development tools"
-
       - run:
           name: Install ISCE requirements
           command: |
             set -ex
             pwd
             mkdir config build install
-            . /opt/conda/bin/activate root
-            conda install --yes cython gdal h5py libgdal pytest numpy fftw scipy scons hdf4 hdf5 libgcc libstdcxx-ng cmake astropy pybind11
-            yum install -y uuid-devel x11-devel motif-devel jq gcc-gfortran opencv opencv-devel opencv-python
-            ln -s /opt/conda/bin/cython /opt/conda/bin/cython3
-            cd /opt/conda/lib
-            unlink libuuid.so
-            unlink libuuid.so.1
-            ln -s /lib64/libuuid.so.1.3.0 libuuid.so
-            ln -s /lib64/libuuid.so.1.3.0 libuuid.so.1
-            cd /lib64
-            test -f libgfortran.so || ln -sv libgfortran.so.*.* libgfortran.so
+            export DEBIAN_FRONTEND=noninteractive
+            apt-get update
+            apt-get install -y scons cython3 git libfftw3-dev libgdal-dev libhdf4-alt-dev libhdf5-dev libmotif-dev libopencv-dev libx11-dev python3-gdal python3-h5py python3-numpy python3-scipy
 
       - run:
           name: Build SConfigISCE and setup dirs
@@ -100,41 +68,36 @@ jobs:
               cd config
               echo "PRJ_SCONS_BUILD = /root/project/build" > SConfigISCE
               echo "PRJ_SCONS_INSTALL = /root/project/install/isce" >> SConfigISCE
-              echo "LIBPATH = /usr/lib64 /usr/lib /opt/conda/lib" >> SConfigISCE
-              python_inc="$(echo /opt/conda/include/python3.* /opt/conda/lib/python3.*/site-packages/numpy/core/include)"
-              echo "CPPPATH = $python_inc /opt/conda/include /usr/include" >> SConfigISCE
-              echo "FORTRANPATH =  /usr/include /opt/conda/include" >> SConfigISCE
+              echo "LIBPATH = /usr/lib64 /usr/lib /usr/lib/x86_64-linux-gnu" >> SConfigISCE
+              python_inc="/usr/include/python3.8 /usr/lib/python3/dist-packages/numpy/core/include"
+              echo "CPPPATH = $python_inc /usr/include /usr/include/gdal /usr/include/opencv4" >> SConfigISCE
+              echo "FORTRANPATH =  /usr/include" >> SConfigISCE
               echo "FORTRAN = /bin/gfortran" >> SConfigISCE
               echo "CC = /bin/gcc" >> SConfigISCE
               echo "CXX = /bin/g++" >> SConfigISCE
               echo "MOTIFLIBPATH = /usr/lib64" >> SConfigISCE
               echo "X11LIBPATH = /usr/lib64" >> SConfigISCE
               echo "MOTIFINCPATH = /usr/include" >> SConfigISCE
               echo "X11INCPATH = /usr/include" >> SConfigISCE
-              echo "RPATH = /opt/conda/lib /usr/lib64 /usr/lib" >> SConfigISCE
+              echo "RPATH = /usr/lib64 /usr/lib" >> SConfigISCE
               cat SConfigISCE
 
       - run:
           name: Build and Install ISCE
           command: |
             set -ex
             pwd
-            . /opt/conda/bin/activate root
             cd src
-            export PATH="/opt/conda/bin:$PATH"
-            export LD_LIBRARY_PATH="/opt/conda/lib:$LD_LIBRARY_PATH"
             SCONS_CONFIG_DIR=/root/project/config scons install --skipcheck
 
       - run:
           name: Test ISCE installation
           command: |
               set -ex
               pwd
-              . /opt/conda/bin/activate root
               ISCE_HOME=/root/project/install/isce
-              export PATH="$ISCE_HOME/bin:$ISCE_HOME/applications:/opt/conda/bin:$PATH"
+              export PATH="$ISCE_HOME/bin:$ISCE_HOME/applications:$PATH"
               export PYTHONPATH="/root/project/install:$PYTHONPATH"
-              export LD_LIBRARY_PATH="/opt/conda/lib:$LD_LIBRARY_PATH"
               topsApp.py --help --steps
               stripmapApp.py --help --steps
               python3 -c "import isce"

CMakeLists.txt

@@ -41,12 +41,17 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_LIST_DIR}")
 endif()
 
 if(NOT DEFINED PYTHON_MODULE_DIR)
-    set(PYTHON_MODULE_DIR packages CACHE PATH
-        "Python module directory (relative to install prefix)")
+    set(PYTHON_MODULE_DIR packages CACHE PATH "Python module directory")
 endif()
 if(NOT DEFINED ISCE2_PKG)
     set(ISCE2_PKG ${PYTHON_MODULE_DIR}/isce2 CACHE PATH
-        "ISCE 2 python package install dir (relative to install prefix)")
+        "ISCE 2 python package install dir")
+endif()
+
+if(IS_ABSOLUTE "${ISCE2_PKG}")
+    set(ISCE2_PKG_FULL "${ISCE2_PKG}")
+else()
+    set(ISCE2_PKG_FULL "${CMAKE_INSTALL_PREFIX}/${ISCE2_PKG}")
 endif()
 
 include(isce2_buildflags)
@@ -95,3 +100,9 @@ else()
 endif()
 install(CODE "execute_process(COMMAND
     ${CMAKE_COMMAND} -E create_symlink ${symsrc} ${symdest})")
+
+# Enable native packaging using CPack
+if(NOT CPACK_PACKAGE_CONTACT)
+    set(CPACK_PACKAGE_CONTACT "Ryan Burns <[email protected]>")
+endif()
+include(CPack)

README.md

@@ -293,7 +293,7 @@ run the following command while in the top directory of the ISCE source (the
 directory containing the SConstruct file):
 
 ```bash
-> rm -rf config.log .sconfig.dblite .sconf_temp
+> rm -rf config.log .sconfig.dblite .sconf_temp .sconsign.dblite
 ```
 
 and then try "scons install" again.

applications/CMakeLists.txt

@@ -32,11 +32,12 @@ install(PROGRAMS ${files}
 
 # Symlink apps into PREFIX/bin so they are on the $PATH
 install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory \
-    ${CMAKE_INSTALL_PREFIX}/bin)"
+    ${CMAKE_INSTALL_FULL_BINDIR})"
     )
+
 foreach(file ${files})
     install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink \
-        ${CMAKE_INSTALL_PREFIX}/${ISCE2_PKG}/applications/${file} \
-        ${CMAKE_INSTALL_PREFIX}/bin/${file})"
+        ${ISCE2_PKG_FULL}/applications/${file} \
+        ${CMAKE_INSTALL_FULL_BINDIR}/${file})"
         )
 endforeach()

components/isceobj/Alos2Proc/Alos2ProcPublic.py

@@ -222,8 +222,8 @@ def readOffset(filename):
     offsets = OffsetField()
     for linex in lines:
         #linexl = re.split('\s+', linex)
-        #detect blank lines with only spaces and tabs
-        if linex.strip() == '':
+        #detect blank lines with only spaces and tabs, lines with invalid numbers
+        if (linex.strip() == '') or ('*' in linex):
             continue
 
         linexl = linex.split()

components/isceobj/Scene/Track.py

@@ -237,7 +237,8 @@ def reAdjustStartLine(self, sortedList, width):
             indx = self.findOverlapLine(sortedList[i-1][1],sortedList[i][1],endLine,width,i-1,i)
             #if indx is not None than indx is the new start line
             #otherwise we use startLine  computed from acquisition time
-            if (indx is not None) and (indx + sortedList[i-1][0] != sortedList[i][0]):
+            #no need to do this for ALOS; otherwise there will be problems when there are multiple prfs and the data are interpolated. C. Liang, 20-dec-2021
+            if (self._frames[0].instrument.platform._mission != 'ALOS') and (indx is not None) and (indx + sortedList[i-1][0] != sortedList[i][0]):
                 startLine.append(indx + sortedList[i-1][0])
                 outputs.append(sortedList[i][1])
                 self.logger.info("Changing starting line for frame %d from %d to %d"%(i,endLine,indx))

components/isceobj/Sensor/ALOS.py

@@ -185,8 +185,10 @@ def populateMetadata(self):
 
         productLevel = float(self.leaderFile.sceneHeaderRecord.metadata[
             'Product level code'])
-        if productLevel == 1.0:
-            self.updateRawParameters()
+
+        #this proves to be not accurate. Adjusting first frame is OK, but adjusting following frames would cause discontinuities between frames. C. Liang, 20-dec-2021
+        # if productLevel == 1.0:
+        #     self.updateRawParameters()
         pass
 
     def _populatePlatform(self):

components/isceobj/Sensor/UAVSAR_Stack.py

@@ -146,14 +146,11 @@ def _populateFrame(self):
         frame.setSensingStart(tStart)
         frame.setSensingStop(tStop)
         frame.setSensingMid(tMid)
-        frame.setNumberOfLines(
-            int(self.metadata['slc_1_1x1_mag.set_rows']))
-        frame.setNumberOfSamples(
-            int(self.metadata['slc_1_1x1_mag.set_cols']))
+        frame.setNumberOfLines(int(self.metadata['slc_{}_1x1_mag.set_rows'.format(self.segment_index)]))
+        frame.setNumberOfSamples(int(self.metadata['slc_{}_1x1_mag.set_cols'.format(self.segment_index)]))
         frame.setPolarization(self.metadata['Polarization'])
-        frame.C0 = self.metadata['slc_1_1x1_mag.col_addr']
-        frame.S0 = self.metadata[
-            'Segment {} Data Starting Azimuth'.format(self.segment_index)]
+        frame.C0 = self.metadata['slc_{}_1x1_mag.col_addr'.format(self.segment_index)]
+        frame.S0 = self.metadata['Segment {} Data Starting Azimuth'.format(self.segment_index)]
         frame.nearLookAngle = self.metadata['Minimum Look Angle']
         frame.farLookAngle = self.metadata['Maximum Look Angle']
         frame.setStartingRange(self.startingRange)
@@ -175,8 +172,7 @@ def _populateFrame(self):
         #of values in degrees at each corner (without rdf unit specified)
         llC = []
         for ic in range(1,5):
-            key = 'Segment {0} Data Approximate Corner {1}'.format(
-                      self.segment_index, ic)
+            key = 'Segment {0} Data Approximate Corner {1}'.format(self.segment_index, ic)
             self.logger.info("key = {}".format(key))
             self.logger.info("metadata[key] = {}".format(self.metadata[key], type(self.metadata[key])))
             llC.append(list(map(float, self.metadata[key].split(','))))
@@ -316,9 +312,9 @@ def extractDoppler(self):
         drho = instrument.getRangePixelSize()  #full res value, not spacing in the dop file
         prf = instrument.getPulseRepetitionFrequency()
         self.logger.info("extractDoppler: rho0, drho, prf = {}, {}, {}".format(rho0, drho, prf))
-        dopfile = self.metadata['dop']
-        f = open(dopfile,'r')
-        x = f.readlines()  #first line is a header
+        dopfile = getattr(self, 'dopplerFile', self.metadata['dop'])
+        with open(dopfile,'r') as f:
+            x = f.readlines()  #first line is a header
 
         import numpy
         z = numpy.array(
@@ -337,21 +333,20 @@ def extractDoppler(self):
         res2 = fit[1][0]  #sum of squared residuals
         self.logger.info("coeffs = {}".format(coefs))
         self.logger.info("rms residual = {}".format(numpy.sqrt(res2/len(dop))))
-        o = open("dop.txt", 'w')
-        for i, d in zip(rhoi, dop):
-            val = polyval(coefs,i)
-            res = d-val
-            o.write("{0} {1} {2} {3}\n".format(i, d, val, res))
-            dopfile = self.metadata['dop']
+        with open("dop.txt", 'w') as o:
+            for i, d in zip(rhoi, dop):
+                val = polyval(coefs,i)
+                res = d-val
+                o.write("{0} {1} {2} {3}\n".format(i, d, val, res))
 
         self.dopplerVals = {'Near':polyval(coefs, 0)}  #need this temporarily in this module
 
         self.logger.info("UAVSAR_Stack.extractDoppler: self.dopplerVals = {}".format(self.dopplerVals))
         self.logger.info("UAVSAR_Stack.extractDoppler: prf = {}".format(prf))
 
-#The doppler file values are in units rad/m.  divide by 2*pi rad/cycle to convert
-#to cycle/m.  Then multiply by velocity to get Hz and divide by prf for dimensionless
-#doppler coefficients
+        #The doppler file values are in units rad/m.  divide by 2*pi rad/cycle to convert
+        #to cycle/m.  Then multiply by velocity to get Hz and divide by prf for dimensionless
+        #doppler coefficients
         dop_scale = self.velocity/2.0/math.pi
         coefs =  [x*dop_scale for x in coefs]
         #Set the coefs in frame._dopplerVsPixel because that is where DefaultDopp looks for them
@@ -361,14 +356,14 @@ def extractDoppler(self):
 
     @property
     def terrainHeight(self):
-#The peg point incorporates the actual terrainHeight
+        #The peg point incorporates the actual terrainHeight
         return 0.0
 
     @property
     def platformHeight(self):
         h = self.metadata['Global Average Altitude']
-#Reduce the platform height by the terrain height because the
-#peg radius of curvature includes the terrain height
+        #Reduce the platform height by the terrain height because the
+        #peg radius of curvature includes the terrain height
         h -= self.metadata['Global Average Terrain Height']
         return h
 

components/isceobj/Sensor/src/ALOS_pre_process/ALOS_pre_process.c

@@ -409,7 +409,7 @@ char   	date[8];
 	    //line number of first line of this prf file
 	    line_number_first = (rspi_new.SC_clock_start[i] - rspi_pre[0].SC_clock_start[0]) * d2s / (1.0 / rspi_pre[0].prf[0]);
 	    //unit: pri of first prf of first image
-	    num_lines_out = (rspi_pre[image_i].frame_counter_end[i] - rspi_pre[image_i].frame_counter_start[i] + 1) * (1.0/rspi_pre[image_i].prf[i]) / (1.0/rspi_pre[0].prf[0]);
+	    num_lines_out = (int)((rspi_pre[image_i].frame_counter_end[i] - rspi_pre[image_i].frame_counter_start[i] + 1) * (1.0/rspi_pre[image_i].prf[i]) / (1.0/rspi_pre[0].prf[0]));
 
         if((fabs(roundfi(line_number_first)-line_number_first)<0.1) && (rspi_pre[image_i].prf[i]==rspi_pre[0].prf[0]))
         	continue;
@@ -469,7 +469,7 @@ char   	date[8];
 	    //append prf i
 	    for(i = 1; i < rspi_new.nPRF; i++){
             //number of lines to be appended between frames if there are gaps
-            num_lines_append = (rspi_new.SC_clock_start[i] - rspi_new.SC_clock_start[0]) * d2s / (1.0/rspi_pre[0].prf[0]) - rspi_new.num_lines[0];
+            num_lines_append = roundfi((rspi_new.SC_clock_start[i] - rspi_new.SC_clock_start[0]) * d2s / (1.0/rspi_pre[0].prf[0])) - rspi_new.num_lines[0];
             if(num_lines_append >= 1){
             	for(j = 0; j < num_lines_append; j++){
             		for(k = 0; k < 2*rspi_new.num_bins[i]; k++)
@@ -485,7 +485,7 @@ char   	date[8];
 	        	die("can't open", rspi_new.input_file[i]);
             num_lines_append = 0;
             for(j = 0; j < rspi_new.num_lines[i]; j++){
-                if((rspi_new.SC_clock_start[i] + j * (1.0/rspi_pre[0].prf[0]) / d2s -  rspi_new.SC_clock_start[0]) * d2s / (1.0/rspi_pre[0].prf[0]) >= rspi_new.num_lines[0]){
+                if(roundfi((rspi_new.SC_clock_start[i] + j * (1.0/rspi_pre[0].prf[0]) / d2s -  rspi_new.SC_clock_start[0]) * d2s / (1.0/rspi_pre[0].prf[0])) >= rspi_new.num_lines[0]){
 			        if(fread((char *)data, 2*sizeof(char)*rspi_new.num_bins[i], 1, next_prf_fp) != 1)
 			        	die("can't read data from", rspi_new.input_file[i]);
 			        if(fwrite((char *)data, 2*sizeof(char)*rspi_new.num_bins[i], 1, first_prf_fp) != 1)

components/isceobj/Sensor/src/ALOS_pre_process/readOrbitPulse.f

@@ -144,7 +144,8 @@ double precision function gete22_15(str)
       end
 
       integer*4 function unpackBytes(i1, i2, i3, i4)
-      integer*4                      i1, i2, i3, i4
-      unpackBytes = iand(i1, 255)*256*256*256 + iand(i2, 255)*256*256 +
-     $              iand(i3, 255)*256         + iand(i4, 255)
+      integer*1                      i1, i2, i3, i4, i255
+      i255 = 255  ! needed to keep gfortran >9 happy
+      unpackBytes = iand(i1, i255)*256*256*256 + iand(i2, i255)*256*256 +
+     $              iand(i3, i255)*256         + iand(i4, i255)
       end function