cctbx-cvs Mailing List for Computational Crystallography Toolbox (Page 9)

Revision: 25576
          http://sourceforge.net/p/cctbx/code/25576
Author:   rjgildea
Date:     2016-10-14 09:20:38 +0000 (Fri, 14 Oct 2016)
Log Message:
-----------
add failing test that demonstrates dials bug #226

Modified Paths:
--------------
    trunk/dxtbx/run_tests.py

Added Paths:
-----------
    trunk/dxtbx/tests/tst_dials_226.py

Modified: trunk/dxtbx/run_tests.py
===================================================================
--- trunk/dxtbx/run_tests.py	2016-10-14 05:36:19 UTC (rev 25575)
+++ trunk/dxtbx/run_tests.py	2016-10-14 09:20:38 UTC (rev 25576)
@@ -9,6 +9,7 @@
     "$D/tests/tst_filecache.py",
     "$D/tests/tst_beamline_definitions.py",
     "$D/tests/tstFormatCBFFull.py",
+    "$D/tests/tst_dials_226.py",
     "$D/tests/command_line/tst_to_xds.py",
     "$D/tests/model/tst_beam.py",
     "$D/tests/model/tst_detector.py",

Added: trunk/dxtbx/tests/tst_dials_226.py
===================================================================
--- trunk/dxtbx/tests/tst_dials_226.py	                        (rev 0)
+++ trunk/dxtbx/tests/tst_dials_226.py	2016-10-14 09:20:38 UTC (rev 25576)
@@ -0,0 +1,25 @@
+from __future__ import division
+from dxtbx.datablock import DataBlockFactory
+import libtbx.load_env
+import os
+
+def run():
+  # https://github.com/dials/dials/issues/226
+
+  dials_regression = libtbx.env.find_in_repositories('dials_regression')
+  if dials_regression is None:
+    print "Skipping tst_dials_226.py: dials_regression not present"
+  h5_path = os.path.join(
+    dials_regression, 'image_examples/SACLA_MPCCD_Cheetah/run266702-0-subset.h5')
+  datablock = DataBlockFactory.from_filenames([h5_path])[0]
+  imageset = datablock.extract_imagesets()[0]
+  path = imageset.get_path(imageset.indices()[0])
+
+  sliced_imageset = imageset[2:]
+  # this line fails
+  path = sliced_imageset.get_path(sliced_imageset.indices()[0])
+
+
+if __name__ == '__main__':
+  run()
+  print 'OK'

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25575
          http://sourceforge.net/p/cctbx/code/25575
Author:   nksauter
Date:     2016-10-14 05:36:19 +0000 (Fri, 14 Oct 2016)
Log Message:
-----------
Refactor cxi.merge, postrefinement code is now in separate module for clearer organization. Attempted to ensure binary identical results, however, transition from python to flex.double() arithmetic for calculating weighted_correlation does affect merging statistics after the 5th significant figure, in some cases.

Modified Paths:
--------------
    trunk/xfel/command_line/cxi_merge.py

Added Paths:
-----------
    trunk/xfel/cxi/postrefinement_legacy_rs.py

Modified: trunk/xfel/command_line/cxi_merge.py
===================================================================
--- trunk/xfel/command_line/cxi_merge.py	2016-10-13 16:00:35 UTC (rev 25574)
+++ trunk/xfel/command_line/cxi_merge.py	2016-10-14 05:36:19 UTC (rev 25575)
@@ -1313,6 +1313,8 @@
       print >> out, "The applied SDFAC is %7.4f"%SDFAC
       corrected_sigmas = observations.sigmas() * SDFAC
       observations = observations.customized_copy(sigmas = corrected_sigmas)
+      observations_original_index = observations_original_index.customized_copy(
+        sigmas = observations_original_index.sigmas() * SDFAC)
 
     print >> out, "Step 6.  Match to reference intensities, filter by correlation, filter out negative intensities."
     assert len(observations_original_index.indices()) \
@@ -1413,11 +1415,12 @@
     data.corr = corr
     data.wavelength = wavelength
 
-    if False and self.params.postrefinement.enable: # for development: choose to comment in this implementation
-      # Refactorization of the Sauter(2015) code; result should be same
-      from cctbx.examples.merging.samosa.per_lattice_postrefinement \
-           import legacy_cxi_merge_postrefinement
-      postx=legacy_cxi_merge_postrefinement(observations_original_index, self.params,
+    if self.params.postrefinement.enable:
+      # Refactorization of the Sauter(2015) code; result should be same to 5 significant figures.
+      # Lack of binary identity is due to the use of Python for old-code weighted correlation,
+      #   contrasted with flex.double arithmetic for new-code.
+      from xfel.cxi.postrefinement_legacy_rs import legacy_rs as postrefinement_algorithm
+      postx=postrefinement_algorithm(observations_original_index, self.params,
            self.i_model, self.miller_set, result, out)
       try:
         postx.run_plain()
@@ -1425,238 +1428,6 @@
       except (AssertionError,ValueError),e:
         return null_data(file_name=file_name, log_out=out.getvalue(), low_signal=True)
 
-    elif self.params.postrefinement.enable:
-      # Original prototype postrefinement from Sauter(2015); assumes mosaicity=0 & monochromatic
-      from scitbx import lbfgs
-      from libtbx import adopt_init_args
-
-      print >> out, "Old correlation is", corr
-      pair1 = flex.int([pair[1] for pair in matches.pairs()])
-      pair0 = flex.int([pair[0] for pair in matches.pairs()])
-      #raw_input("go:")
-      # narrow things down to the set that matches, only
-      observations = observations.customized_copy(
-        indices = flex.miller_index([observations.indices()[p] for p in pair1]),
-        data = flex.double([observations.data()[p] for p in pair1]),
-        sigmas = flex.double([observations.sigmas()[p] for p in pair1]),
-      )
-      observations_original_index = observations_original_index.customized_copy(
-        indices = flex.miller_index([observations_original_index.indices()[p] for p in pair1]),
-        data = flex.double([observations_original_index.data()[p] for p in pair1]),
-        sigmas = flex.double([observations_original_index.sigmas()[p] for p in pair1]),
-      )
-
-      #IOBSVEC = flex.double([observations.data()[p] for p in pair1])
-      #MILLER = flex.miller_index([observations_original_index.indices()[p] for p in pair1])
-      IOBSVEC = observations.data()
-      ICALCVEC = flex.double([self.i_model.data()[p] for p in pair0])
-      MILLER = observations_original_index.indices()
-      ORI = result["current_orientation"][0]
-      Astar = matrix.sqr(ORI.reciprocal_matrix())
-      WAVE = result["wavelength"]
-      BEAM = matrix.col((0.0,0.0,-1./WAVE))
-      BFACTOR = 0.
-      DSSQ = ORI.unit_cell().d_star_sq(MILLER)
-
-      if self.params.postrefinement.algorithm=="rs":
-        Rhall = flex.double()
-        for mill in MILLER:
-          H = matrix.col(mill)
-          Xhkl = Astar*H
-          Rh = ( Xhkl + BEAM ).length() - (1./WAVE)
-          Rhall.append(Rh)
-        Rs = math.sqrt(flex.mean(Rhall*Rhall))
-
-        RS = 1./10000. # reciprocal effective domain size of 1 micron
-        RS = Rs        # try this empirically determined approximate, monochrome, a-mosaic value
-        current = flex.double([slope, BFACTOR, RS, 0., 0.])
-
-        class unpack(object):
-         def __init__(YY,values):
-          YY.reference = values # simply the flex double list of parameters
-         def __getattr__(YY,item):
-          if item=="thetax" : return YY.reference[3]
-          if item=="thetay" : return YY.reference[4]
-          if item=="G" :      return YY.reference[0]
-          if item=="BFACTOR": return YY.reference[1]
-          if item=="RS":      return YY.reference[2]
-          return getattr(YY,item)
-
-         def show(values):
-          print >> out, "G: %10.7f"%values.G,
-          print >> out, "B: %10.7f"%values.BFACTOR, \
-                "RS: %10.7f"%values.RS, \
-                "%7.3f deg %7.3f deg"%(
-            180.*values.thetax/math.pi,180.*values.thetay/math.pi)
-
-        def lorentz_callable(values):
-          return get_partiality_array(values)
-
-        def get_partiality_array(values):
-          rs = values.RS
-          Rh = get_Rh_array(values)
-          rs_sq = rs*rs
-          PB = rs_sq / ((2. * (Rh * Rh)) + rs_sq)
-          return PB
-
-      elif self.params.postrefinement.algorithm=="eta_deff":
-        DVEC = ORI.unit_cell().d(MILLER)
-        eta_init = 2. * result["ML_half_mosaicity_deg"][0] * math.pi/180.
-        D_eff_init = 2.*result["ML_domain_size_ang"][0]
-        current = flex.double([slope, BFACTOR, eta_init, 0., 0.,D_eff_init,])
-
-        class unpack(object):
-         def __init__(YY,values):
-          YY.reference = values # simply the flex double list of parameters
-         def __getattr__(YY,item):
-          if item=="thetax" : return YY.reference[3]
-          if item=="thetay" : return YY.reference[4]
-          if item=="G" :      return YY.reference[0]
-          if item=="BFACTOR": return YY.reference[1]
-          if item=="ETA":      return YY.reference[2]
-          if item=="DEFF":      return YY.reference[5]
-          return getattr(YY,item)
-
-         def show(values):
-          print >> out, "%10.7f"%values.G,
-          print >> out, "%10.7f"%values.BFACTOR, \
-                "eta %10.7f"%values.ETA, \
-                "Deff %10.2f"%values.DEFF, \
-                "%7.3f deg %7.3f deg"%(
-            180.*values.thetax/math.pi,180.*values.thetay/math.pi)
-
-        def lorentz_callable(values):
-          Rh = get_Rh_array(values)
-          Rs = flex.double(len(MILLER),1./values.DEFF)+flex.double(len(MILLER),values.ETA/2.)/DVEC
-          ratio = Rh / Rs
-          ratio_abs = flex.abs(ratio)
-          return ratio_abs
-
-        def get_partiality_array(values):
-          Rh = get_Rh_array(values)
-          Rs = flex.double(len(MILLER),1./values.DEFF)+flex.double(len(MILLER),values.ETA/2.)/DVEC
-          Rs_sq = Rs * Rs
-          Rh_sq = Rh * Rh
-          numerator = Rs_sq - Rh_sq
-          denominator = values.DEFF * Rs * Rs_sq
-          partiality = numerator / denominator
-          return partiality
-
-      def get_Rh_array(values):
-        Rh = flex.double()
-        eff_Astar = get_eff_Astar(values)
-        for mill in MILLER:
-          x = eff_Astar * matrix.col(mill)
-          Svec = x + BEAM
-          Rh.append(Svec.length() - (1./WAVE))
-        return Rh
-
-      def get_eff_Astar(values):
-        thetax = values.thetax; thetay = values.thetay;
-        effective_orientation = ORI.rotate_thru((1,0,0),thetax
-           ).rotate_thru((0,1,0),thetay
-           )
-        return matrix.sqr(effective_orientation.reciprocal_matrix())
-
-      def scaler_callable(values):
-        PB = get_partiality_array(values)
-        EXP = flex.exp(-2.*values.BFACTOR*DSSQ)
-        terms = values.G * EXP * PB
-        return terms
-
-      def fvec_callable(values):
-        PB = get_partiality_array(values)
-        EXP = flex.exp(-2.*values.BFACTOR*DSSQ)
-        terms = (values.G * EXP * PB * ICALCVEC - IOBSVEC)
-        # Ideas for improvement
-        #   straightforward to also include sigma weighting
-        #   add extra terms representing rotational excursion: terms.concatenate(1.e7*Rh)
-        return terms
-
-      func = fvec_callable(unpack(current))
-      functional = flex.sum(func*func)
-      print >> out, "functional",functional
-
-      class c_minimizer:
-
-        def __init__(self, current_x=None,
-                     min_iterations=0, max_calls=1000, max_drop_eps=1.e-5):
-          adopt_init_args(self, locals())
-          self.n = current_x.size()
-          self.x = current_x
-          self.minimizer = lbfgs.run(
-            target_evaluator=self,
-            termination_params=lbfgs.termination_parameters(
-              traditional_convergence_test=False,
-              drop_convergence_test_max_drop_eps=max_drop_eps,
-              min_iterations=min_iterations,
-              max_iterations = None,
-              max_calls=max_calls),
-            exception_handling_params=lbfgs.exception_handling_parameters(
-               ignore_line_search_failed_rounding_errors=True,
-               ignore_line_search_failed_step_at_lower_bound=True,#the only change from default
-               ignore_line_search_failed_step_at_upper_bound=False,
-               ignore_line_search_failed_maxfev=False,
-               ignore_line_search_failed_xtol=False,
-               ignore_search_direction_not_descent=False)
-            )
-
-        def compute_functional_and_gradients(self):
-          values = unpack(self.x)
-          assert -150. < values.BFACTOR < 150. # limits on the exponent, please
-          self.func = fvec_callable(values)
-          functional = flex.sum(self.func*self.func)
-          self.f = functional
-          DELTA = 1.E-7
-          self.g = flex.double()
-          for x in xrange(self.n):
-            templist = list(self.x)
-            templist[x]+=DELTA
-            dvalues = flex.double(templist)
-
-            dfunc = fvec_callable(unpack(dvalues))
-            dfunctional = flex.sum(dfunc*dfunc)
-            #calculate by finite_difference
-            self.g.append( ( dfunctional-functional )/DELTA )
-          self.g[2]=0.
-          print >> out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)),
-          values.show()
-          return self.f, self.g
-
-        def __del__(self):
-          values = unpack(self.x)
-          print >> out, "FINALMODEL",
-          print >> out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)),
-          values.show()
-
-      try:
-        MINI = c_minimizer( current_x = current )
-      except AssertionError: # on exponential overflow
-        return null_data(
-               file_name=file_name, log_out=out.getvalue(), low_signal=True)
-      scaler = scaler_callable(unpack(MINI.x))
-      if self.params.postrefinement.algorithm=="rs":
-        fat_selection = (lorentz_callable(unpack(MINI.x)) > 0.2)
-      else:
-        fat_selection = (lorentz_callable(unpack(MINI.x)) < 0.9)
-      fat_count = fat_selection.count(True)
-
-      #avoid empty database INSERT, if there are insufficient centrally-located Bragg spots:
-      if fat_count < 3:
-        return null_data(
-               file_name=file_name, log_out=out.getvalue(), low_signal=True)
-      print >> out, "On total %5d the fat selection is %5d"%(len(observations.indices()), fat_count)
-      observations_original_index = observations_original_index.select(fat_selection)
-
-      observations = observations.customized_copy(
-        indices = observations.indices().select(fat_selection),
-        data = (observations.data()/scaler).select(fat_selection),
-        sigmas = (observations.sigmas()/scaler).select(fat_selection)
-      )
-      matches = miller.match_multi_indices(
-        miller_indices_unique=self.miller_set.indices(),
-        miller_indices=observations.indices())
-
     if not self.params.scaling.enable or self.params.postrefinement.enable: # Do not scale anything
       print >> out, "Scale factor to an isomorphous reference PDB will NOT be applied."
       slope = 1.0

Added: trunk/xfel/cxi/postrefinement_legacy_rs.py
===================================================================
--- trunk/xfel/cxi/postrefinement_legacy_rs.py	                        (rev 0)
+++ trunk/xfel/cxi/postrefinement_legacy_rs.py	2016-10-14 05:36:19 UTC (rev 25575)
@@ -0,0 +1,344 @@
+from __future__ import division
+import math
+from scitbx import matrix
+from cctbx import miller
+from dials.array_family import flex
+from scitbx.math.tests.tst_weighted_correlation import simple_weighted_correlation
+from libtbx import adopt_init_args, group_args
+
+class legacy_rs(object):
+  def __init__(self,measurements_orig, params, i_model, miller_set, result, out):
+    measurements = measurements_orig.deep_copy()
+
+    # Now manipulate the data to conform to unit cell, asu, and space group
+    # of reference.  The resolution will be cut later.
+    # Only works if there is NOT an indexing ambiguity!
+    observations = measurements.customized_copy(
+      anomalous_flag=not params.merge_anomalous,
+      crystal_symmetry=miller_set.crystal_symmetry()
+      ).map_to_asu()
+
+    observations_original_index = measurements.customized_copy(
+      anomalous_flag=not params.merge_anomalous,
+      crystal_symmetry=miller_set.crystal_symmetry()
+      )
+
+    # Ensure that match_multi_indices() will return identical results
+    # when a frame's observations are matched against the
+    # pre-generated Miller set, self.miller_set, and the reference
+    # data set, self.i_model.  The implication is that the same match
+    # can be used to map Miller indices to array indices for intensity
+    # accumulation, and for determination of the correlation
+    # coefficient in the presence of a scaling reference.
+
+    assert len(i_model.indices()) == len(miller_set.indices()) \
+        and  (i_model.indices() ==
+              miller_set.indices()).count(False) == 0
+
+    matches = miller.match_multi_indices(
+      miller_indices_unique=miller_set.indices(),
+      miller_indices=observations.indices())
+
+    pair1 = flex.int([pair[1] for pair in matches.pairs()])
+    pair0 = flex.int([pair[0] for pair in matches.pairs()])
+    # narrow things down to the set that matches, only
+    observations_pair1_selected = observations.customized_copy(
+      indices = flex.miller_index([observations.indices()[p] for p in pair1]),
+      data = flex.double([observations.data()[p] for p in pair1]),
+      sigmas = flex.double([observations.sigmas()[p] for p in pair1]),
+    )
+    observations_original_index_pair1_selected = observations_original_index.customized_copy(
+      indices = flex.miller_index([observations_original_index.indices()[p] for p in pair1]),
+      data = flex.double([observations_original_index.data()[p] for p in pair1]),
+      sigmas = flex.double([observations_original_index.sigmas()[p] for p in pair1]),
+    )
+###################
+    I_observed = observations_pair1_selected.data()
+    MILLER = observations_original_index_pair1_selected.indices()
+    ORI = result["current_orientation"][0]
+    Astar = matrix.sqr(ORI.reciprocal_matrix())
+    WAVE = result["wavelength"]
+    BEAM = matrix.col((0.0,0.0,-1./WAVE))
+    BFACTOR = 0.
+
+    #calculation of correlation here
+    I_reference = flex.double([i_model.data()[pair[0]] for pair in matches.pairs()])
+    use_weights = False # New facility for getting variance-weighted correlation
+
+    if use_weights:
+       #variance weighting
+      I_weight = flex.double(
+        [1./(observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs()])
+    else:
+      I_weight = flex.double(len(observations_pair1_selected.sigmas()), 1.)
+
+    """Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement:
+       include_negatives = True
+       + and - reflections both used for Rh distribution for initial estimate of RS parameter
+       + and - reflections both used for calc/obs correlation slope for initial estimate of G parameter
+       + and - reflections both passed to the refinery and used in the target function (makes sense if
+                           you look at it from a certain point of view)
+
+       include_negatives = False
+       + and - reflections both used for Rh distribution for initial estimate of RS parameter
+       +       reflections only used for calc/obs correlation slope for initial estimate of G parameter
+       + and - reflections both passed to the refinery and used in the target function (makes sense if
+                           you look at it from a certain point of view)
+    """
+    if params.include_negatives:
+      SWC = simple_weighted_correlation(I_weight, I_reference, I_observed)
+    else:
+      non_positive = ( observations_pair1_selected.data() <= 0 )
+      SWC = simple_weighted_correlation(I_weight.select(~non_positive),
+            I_reference.select(~non_positive), I_observed.select(~non_positive))
+
+    print >> out, "Old correlation is", SWC.corr
+    if params.postrefinement.algorithm=="rs":
+      Rhall = flex.double()
+      for mill in MILLER:
+        H = matrix.col(mill)
+        Xhkl = Astar*H
+        Rh = ( Xhkl + BEAM ).length() - (1./WAVE)
+        Rhall.append(Rh)
+      Rs = math.sqrt(flex.mean(Rhall*Rhall))
+
+      RS = 1./10000. # reciprocal effective domain size of 1 micron
+      RS = Rs        # try this empirically determined approximate, monochrome, a-mosaic value
+      current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.])
+
+      parameterization_class = rs_parameterization
+      refinery = rs_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE,
+        ICALCVEC = I_reference, IOBSVEC = I_observed)
+
+    elif params.postrefinement.algorithm=="eta_deff":
+      eta_init = 2. * result["ML_half_mosaicity_deg"][0] * math.pi/180.
+      D_eff_init = 2.*result["ML_domain_size_ang"][0]
+      current = flex.double([SWC.slope, BFACTOR, eta_init, 0., 0.,D_eff_init,])
+
+      parameterization_class = eta_deff_parameterization
+      refinery = eta_deff_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE,
+        ICALCVEC = I_reference, IOBSVEC = I_observed)
+
+    func = refinery.fvec_callable(parameterization_class(current))
+    functional = flex.sum(func*func)
+    print >> out, "functional",functional
+    self.current = current; self.parameterization_class = parameterization_class
+    self.refinery = refinery; self.out=out; self.params = params;
+    self.miller_set = miller_set
+    self.observations_pair1_selected = observations_pair1_selected;
+    self.observations_original_index_pair1_selected = observations_original_index_pair1_selected
+
+  def run_plain(self):
+    self.MINI = lbfgs_minimizer_base( current_x = self.current,
+        parameterization = self.parameterization_class, refinery = self.refinery,
+        out = self.out )
+
+  def result_for_cxi_merge(self, file_name):
+    scaler = self.refinery.scaler_callable(self.parameterization_class(self.MINI.x))
+    if self.params.postrefinement.algorithm=="rs":
+      fat_selection = (self.refinery.lorentz_callable(self.parameterization_class(self.MINI.x)) > 0.2)
+    else:
+      fat_selection = (self.refinery.lorentz_callable(self.parameterization_class(self.MINI.x)) < 0.9)
+    fat_count = fat_selection.count(True)
+
+    #avoid empty database INSERT, if insufficient centrally-located Bragg spots:
+    # in samosa, handle this at a higher level, but handle it somehow.
+    if fat_count < 3:
+      raise ValueError
+    print >> self.out, "On total %5d the fat selection is %5d"%(
+      len(self.observations_pair1_selected.indices()), fat_count)
+    observations_original_index = \
+      self.observations_original_index_pair1_selected.select(fat_selection)
+
+    observations = self.observations_pair1_selected.customized_copy(
+      indices = self.observations_pair1_selected.indices().select(fat_selection),
+      data = (self.observations_pair1_selected.data()/scaler).select(fat_selection),
+      sigmas = (self.observations_pair1_selected.sigmas()/scaler).select(fat_selection)
+    )
+    matches = miller.match_multi_indices(
+      miller_indices_unique=self.miller_set.indices(),
+      miller_indices=observations.indices())
+    return observations_original_index,observations,matches
+
+  def result_for_samosa(self):
+    values = self.parameterization_class(self.MINI.x)
+    return self.refinery.get_eff_Astar(values), values.RS
+
+class refinery_base(group_args):
+    def __init__(self, **kwargs):
+      group_args.__init__(self,**kwargs)
+      mandatory = ["ORI","MILLER","BEAM","WAVE","ICALCVEC","IOBSVEC"]
+      for key in mandatory: getattr(self,key)
+      self.DSSQ = self.ORI.unit_cell().d_star_sq(self.MILLER)
+
+    """Refinery class takes reference and observations, and implements target
+    functions and derivatives for a particular model paradigm."""
+    def get_Rh_array(self, values):
+      Rh = flex.double()
+      eff_Astar = self.get_eff_Astar(values)
+      for mill in self.MILLER:
+        x = eff_Astar * matrix.col(mill)
+        Svec = x + self.BEAM
+        Rh.append(Svec.length() - (1./self.WAVE))
+      return Rh
+
+    def get_s1_array(self, values):
+      miller_vec = self.MILLER.as_vec3_double()
+      ref_ori = matrix.sqr(self.ORI.reciprocal_matrix())
+      Rx = matrix.col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(values.thetax)
+      Ry = matrix.col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(values.thetay)
+      s_array = flex.mat3_double(len(self.MILLER),Ry * Rx * ref_ori) * miller_vec
+      s1_array = s_array + flex.vec3_double(len(self.MILLER), self.BEAM)
+      return s1_array
+
+    def get_eff_Astar(self, values):
+      thetax = values.thetax; thetay = values.thetay;
+      effective_orientation = self.ORI.rotate_thru((1,0,0),thetax
+         ).rotate_thru((0,1,0),thetay
+         )
+      return matrix.sqr(effective_orientation.reciprocal_matrix())
+
+    def scaler_callable(self, values):
+      PB = self.get_partiality_array(values)
+      EXP = flex.exp(-2.*values.BFACTOR*self.DSSQ)
+      terms = values.G * EXP * PB
+      return terms
+
+    def fvec_callable(self, values):
+      PB = self.get_partiality_array(values)
+      EXP = flex.exp(-2.*values.BFACTOR*self.DSSQ)
+      terms = (values.G * EXP * PB * self.ICALCVEC - self.IOBSVEC)
+      # Ideas for improvement
+      #   straightforward to also include sigma weighting
+      #   add extra terms representing rotational excursion: terms.concatenate(1.e7*Rh)
+      return terms
+
+class rs_refinery(refinery_base):
+    def lorentz_callable(self,values):
+      return self.get_partiality_array(values)
+
+    def get_partiality_array(self,values):
+      rs = values.RS
+      Rh = self.get_Rh_array(values)
+      rs_sq = rs*rs
+      PB = rs_sq / ((2. * (Rh * Rh)) + rs_sq)
+      return PB
+
+class eta_deff_refinery(refinery_base):
+    def __init__(self, **kwargs):
+      refinery_base.__init__(self,**kwargs)
+      self.DVEC = self.ORI.unit_cell().d(self.MILLER)
+
+    def lorentz_callable(self,values):
+      Rh = self.get_Rh_array(values)
+      Rs = flex.double(len(self.MILLER),1./values.DEFF)+flex.double(len(self.MILLER),values.ETA/2.)/self.DVEC
+      ratio = Rh / Rs
+      ratio_abs = flex.abs(ratio)
+      return ratio_abs
+
+    def get_partiality_array(self,values):
+      Rh = self.get_Rh_array(values)
+      Rs = flex.double(len(self.MILLER),1./values.DEFF)+flex.double(len(self.MILLER),values.ETA/2.)/self.DVEC
+      Rs_sq = Rs * Rs
+      Rh_sq = Rh * Rh
+      numerator = Rs_sq - Rh_sq
+      denominator = values.DEFF * Rs * Rs_sq
+      partiality = numerator / denominator
+      return partiality
+
+class unpack_base(object):
+  "abstract interface"
+  def __init__(YY,values):
+    YY.reference = values # simply the flex double list of parameters
+  def __getattr__(YY,item):
+    raise NotImplementedError
+  def show(values,out):
+    raise NotImplementedError
+
+class rs_parameterization(unpack_base):
+  def __getattr__(YY,item):
+    if item=="thetax" : return YY.reference[3]
+    if item=="thetay" : return YY.reference[4]
+    if item=="G" :      return YY.reference[0]
+    if item=="BFACTOR": return YY.reference[1]
+    if item=="RS":      return YY.reference[2]
+    return getattr(YY,item)
+
+  def show(YY, out):
+    print >> out, "G: %10.7f"%YY.G,
+    print >> out, "B: %10.7f"%YY.BFACTOR, \
+        "RS: %10.7f"%YY.RS, \
+        "%7.3f deg %7.3f deg"%(
+        180.*YY.thetax/math.pi,180.*YY.thetay/math.pi)
+
+class eta_deff_parameterization(unpack_base):
+  def __getattr__(YY,item):
+    if item=="thetax" : return YY.reference[3]
+    if item=="thetay" : return YY.reference[4]
+    if item=="G" :      return YY.reference[0]
+    if item=="BFACTOR": return YY.reference[1]
+    if item=="ETA":      return YY.reference[2]
+    if item=="DEFF":      return YY.reference[5]
+    return getattr(YY,item)
+
+  def show(YY, out):
+    print >> out, "%10.7f"%YY.G,
+    print >> out, "%10.7f"%YY.BFACTOR, \
+          "eta %10.7f"%YY.ETA, \
+          "Deff %10.2f"%YY.DEFF, \
+          "%7.3f deg %7.3f deg"%(
+      180.*YY.thetax/math.pi,180.*YY.thetay/math.pi)
+
+class lbfgs_minimizer_base:
+
+  def __init__(self, current_x=None, parameterization=None, refinery=None, out=None,
+               min_iterations=0, max_calls=1000, max_drop_eps=1.e-5):
+    adopt_init_args(self, locals())
+    self.n = current_x.size()
+    self.x = current_x
+    from scitbx import lbfgs
+    self.minimizer = lbfgs.run(
+      target_evaluator=self,
+      termination_params=lbfgs.termination_parameters(
+        traditional_convergence_test=False,
+        drop_convergence_test_max_drop_eps=max_drop_eps,
+        min_iterations=min_iterations,
+        max_iterations = None,
+        max_calls=max_calls),
+      exception_handling_params=lbfgs.exception_handling_parameters(
+         ignore_line_search_failed_rounding_errors=True,
+         ignore_line_search_failed_step_at_lower_bound=True,#the only change from default
+         ignore_line_search_failed_step_at_upper_bound=False,
+         ignore_line_search_failed_maxfev=False,
+         ignore_line_search_failed_xtol=False,
+         ignore_search_direction_not_descent=False)
+      )
+
+  def compute_functional_and_gradients(self):
+    values = self.parameterization(self.x)
+    assert -150. < values.BFACTOR < 150. # limits on the exponent, please
+    self.func = self.refinery.fvec_callable(values)
+    functional = flex.sum(self.func*self.func)
+    self.f = functional
+    DELTA = 1.E-7
+    self.g = flex.double()
+    for x in xrange(self.n):
+      templist = list(self.x)
+      templist[x]+=DELTA
+      dvalues = flex.double(templist)
+
+      dfunc = self.refinery.fvec_callable(self.parameterization(dvalues))
+      dfunctional = flex.sum(dfunc*dfunc)
+      #calculate by finite_difference
+      self.g.append( ( dfunctional-functional )/DELTA )
+    self.g[2]=0.
+    print >> self.out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)),
+    values.show(self.out)
+    return self.f, self.g
+
+  def __del__(self):
+    values = self.parameterization(self.x)
+    print >> self.out, "FINALMODEL",
+    print >> self.out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)),
+    values.show(self.out)
+

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25574
          http://sourceforge.net/p/cctbx/code/25574
Author:   phyy-nx
Date:     2016-10-13 16:00:35 +0000 (Thu, 13 Oct 2016)
Log Message:
-----------
Latest changes from Takanori

Modified Paths:
--------------
    trunk/dxtbx/format/FormatHDF5SaclaMPCCD.py

Modified: trunk/dxtbx/format/FormatHDF5SaclaMPCCD.py
===================================================================
--- trunk/dxtbx/format/FormatHDF5SaclaMPCCD.py	2016-10-12 22:22:43 UTC (rev 25573)
+++ trunk/dxtbx/format/FormatHDF5SaclaMPCCD.py	2016-10-13 16:00:35 UTC (rev 25574)
@@ -1,100 +1,152 @@
 from __future__ import division
-from dxtbx.format.Format import Format
 from dxtbx.format.FormatHDF5 import FormatHDF5
 from dxtbx.format.FormatStill import FormatStill
 
+# 151028: deepcopying this class causes crash in h5py
+#         temporary fix by closing the file in every methods(!)
+# 161003: updated to follow dxtbx changes
+#         removed iotbx support, which was incomplete anyway
+# 161005: get wavelength from the file
+
 class FormatHDF5SaclaMPCCD(FormatHDF5, FormatStill):
   '''
   Class to handle multi-event HDF5 files from MPCCD
   preprocessed by Cheetah SFX pipeline at SACLA.
-
   To handle reassembled images from "DataConvert3 -reconst"
   (old pipeline), use FormatHDF5Sacla.
+  To override metrology, use the following environmental variables.
+     MPCCD_GEOMETRY, MPCCD_DISTANCE
+     MPCCD_RECONST_MODE
+
+  You can also specify reference_geometry in dials.stills_process.
   '''
 
   @staticmethod
+  def split_tag(str):
+    splitted = str.split("//")
+    if (len(splitted) == 2):
+      return (splitted[0], splitted[1])
+    else:
+      return (splitted[0], None)
+
+  @staticmethod
   def understand(image_file):
     import h5py
-    h5_handle = h5py.File(image_file, 'r')
 
-    for elem in h5_handle:
-      if elem.startswith("tag-"):
-        return True
+    image, tag = FormatHDF5SaclaMPCCD.split_tag(image_file)
+    h5_handle = h5py.File(image, 'r')
 
-    return False
+    if tag is None:
+      for elem in h5_handle:
+        if elem.startswith("tag-"):
+          return True
+      return False
+    else:
+      return (tag in h5_handle)
 
-  def __init__(self, image_file):
+  def __init__(self, image_file, index = 0, reconst_mode = False):
     assert(self.understand(image_file))
-    FormatHDF5.__init__(self, image_file)
+    image, tag = self.split_tag(image_file)
     self._raw_data = None
+    self.index = index
+    self.tag = tag
+    self.image_filename = image
+    FormatHDF5.__init__(self, image)
 
-    self.pixel_size = 50 / 1000 # 50 um
+    self.PIXEL_SIZE = 50 / 1000 # 50 um
     self.RECONST_SIZE = 2398 # compatible with DataConvert3 -reconst mode
 
-    self.RECONST_MODE = True
-    self.RECONST_64 = True
-
-    self.distance = 52.0 # mm
-
-    # TODO: These should be read from geometry file.
+    # These hard-coded values can be overwritten
+    # by MPCCD_GEOMETRY and MPCCD_DISTANCE
+    self.distance = 50.0 # mm
     self.panel_origins = [(-1755.000000, 51711.000000, 0.000000),
-                     (-1711.000000, 24944.000000, 0.000000),
-                     (817.000000, -1808.000000, 0.000000),
-                     (812.000000, -28466.000000, 0.000000),
-                     (-792.000000, 28544.000000, 0.000000),
-                     (-781.000000, 1840.000000, 0.000000),
-                     (1650.000000, -24900.000000, 0.000000),
-                     (1655.000000, -51626.000000, 0.000000)] # um
+                          (-1711.000000, 24944.000000, 0.000000),
+                          (817.000000, -1808.000000, 0.000000),
+                          (812.000000, -28466.000000, 0.000000),
+                          (-792.000000, 28544.000000, 0.000000),
+                          (-781.000000, 1840.000000, 0.000000),
+                          (1650.000000, -24900.000000, 0.000000),
+                          (1655.000000, -51626.000000, 0.000000)] # um
     self.panel_rotations = [-89.906197, -89.915802, -89.980003, -89.929298,
-                       89.963097, 89.880798, 90.000000, 90.029503]
+                            89.963097, 89.880798, 90.000000, 90.029503]
 
+    import os
+
+    if 'MPCCD_RECONST_MODE' in os.environ:
+      reconst_mode = bool(os.environ['MPCCD_RECONST_MODE'])
+    self.RECONST_MODE = reconst_mode
+    self.RECONST_64 = True # Set False if you want to keep panels completely horizontal
+                           # But this makes errors bigger.
+
+    if 'MPCCD_GEOMETRY' in os.environ:
+      try:
+        tmp = map(float, os.environ['MPCCD_GEOMETRY'].split(","))
+        if len(tmp) != 24:
+          raise
+        for i in range(8):
+          self.panel_origins[i] = (-tmp[i * 3], tmp[i * 3 + 1], 0)
+          self.panel_rotations[i] = tmp[i * 3 + 2]
+      except Exception:
+        raise "Invalid MPCCD Geomtry"
+    if 'MPCCD_DISTANCE' in os.environ:
+      self.distance = float(os.environ['MPCCD_DISTANCE'])
+
   def _start(self):
     import h5py
-    self._h5_handle = h5py.File(self.get_image_file(), 'r')
+    h5_handle = h5py.File(self.image_filename, 'r')
 
-    self._images = [tag for tag in self._h5_handle if tag.startswith("tag-")]
-    self.tag = self._images[0]
+    self._images = [tag for tag in h5_handle if tag.startswith("tag-")]
+    if self.tag is None:
+      self.tag = self._images[self.index]
+    h5_handle.close()
 
+  def get_image_file(self, index=None):
+    return self.image_filename
+
+  def set_index(self, index):
+    assert(index < len(self._images))
+
+    self.index = index
+    self.tag = self._images[self.index]
+    self._raw_data = None
+
   def _detector(self, index=None):
-    from dxtbx.model import Detector
+    from dxtbx.model.detector import Detector
     from scitbx import matrix
     import math
 
-    # Ignore this for now.
-    # TODO: must consider when a new MPCCD with 300um thickness
-    #       is released.
+    wavelength = self.get_beam(index).get_wavelength()
 
-    self.wavelength = 1.77 # A, this must be got from _beam
-    #    thickness = 60 # um
-    #
-    #    from cctbx.eltbx import attenuation_coefficient
-    #    table = attenuation_coefficient.get_table("Si")
-    #    mu = table.mu_at_angstrom(wavelength) / 10.0
-    #    t0 = thickness
-    #    px_mm = ParallaxCorrectedPxMmStrategy(mu, t0)
+    from dxtbx.model import ParallaxCorrectedPxMmStrategy
+    t0 = 0.050  # sensor thickness in um
+    from cctbx.eltbx import attenuation_coefficient
+    table = attenuation_coefficient.get_table("Si")
+    mu = table.mu_at_angstrom(wavelength) / 10.0
+    px_mm = ParallaxCorrectedPxMmStrategy(mu, t0)
 
-    detector = Detector()
-    root = detector.hierarchy()
-    root.set_frame(
-      (1, 0, 0),
-      (0, 1, 0),
-      (0, 0, - self.distance))
-
     if self.RECONST_MODE:
       return self._detector_factory.simple(
         sensor = 'PAD',
         distance = self.distance,
-        beam_centre = (self.RECONST_SIZE * self.pixel_size / 2,
-                       self.RECONST_SIZE * self.pixel_size / 2),
+        beam_centre = (self.RECONST_SIZE / 2 * self.PIXEL_SIZE,
+                       self.RECONST_SIZE / 2 * self.PIXEL_SIZE),
         fast_direction = '+x',
         slow_direction = '-y',
-        pixel_size = (self.pixel_size,
-                      self.pixel_size),
+        PIXEL_SIZE = (self.PIXEL_SIZE,
+                      self.PIXEL_SIZE),
         image_size = (self.RECONST_SIZE,
                       self.RECONST_SIZE),
         trusted_range = (-1, 1000000),
-        mask = [])  # a list of dead rectangles
+        px_mm = px_mm,
+        mask = [])  # TODO: add gaps
 
+    detector = Detector()
+    root = detector.hierarchy()
+    root.set_frame(
+      (1, 0, 0),
+      (0, 1, 0),
+      (0, 0, - self.distance))
+
     for i in range(8):
       angle = math.pi * self.panel_rotations[i] / 180.0
       fast = matrix.col((math.cos(angle), math.sin(angle), 0))
@@ -104,80 +156,93 @@
       origin = matrix.col((-self.panel_origins[i][0],
                             self.panel_origins[i][1],
                             self.panel_origins[i][2])) / 1000.0
-
-      # OBS! you need to set the panel to a root before set local frame...
       p = root.add_panel()
-      p.set_name('panel-%01d' % i)
+      p.set_type("SENSOR_PAD")
+      p.set_name('Panel%d' % i)
       p.set_image_size((512, 1024))
       p.set_trusted_range((-1, 1000000))
-      p.set_pixel_size((self.pixel_size, self.pixel_size))
+      p.set_pixel_size((self.PIXEL_SIZE, self.PIXEL_SIZE))
+      p.set_thickness(t0)
       p.set_local_frame(
         fast.elems,
         slow.elems,
         origin.elems)
-      xmin, ymin = 0, i * 1024
-      p.set_raw_image_offset((xmin, ymin))
-      # p.set_px_mm_strategy(px_mm)
+      p.set_px_mm_strategy(px_mm)
+      p.set_gain(10)
 
     return detector
 
-  def _beam(self, index=None):
-    eV = self._h5_handle[self.tag]['photon_energy_ev'].value
+  def _beam(self):
+    import h5py
+    h5_handle = h5py.File(self.image_filename, 'r')
+    eV = h5_handle[self.tag]['photon_energy_ev'].value
+    h5_handle.close()
 
-    return self._beam_factory.simple(12398.4/eV)
+    return self._beam_factory.simple(12398.4 / eV)
 
   def get_num_images(self):
     return len(self._images)
 
-  def get_raw_data(self, index=0):
+  def get_raw_data(self, index=None):
     import numpy
 
-    self.tag = self._images[index]
+    if index is not None and self.index != index:
+      self.set_index(index)
 
     if self._raw_data is None:
       from scitbx.array_family import flex
 
       if self.RECONST_MODE:
-        return flex.int(self.reconst_image())
+        self._raw_data = flex.int(self.reconst_image())
 
       else:
-        print "get_raw_data(%d) for %s" % (index, self.tag)
-        data = self._h5_handle[self.tag]["data"][()].astype(numpy.int32)
+        import h5py
+        h5_handle = h5py.File(self.image_filename, 'r')
+
+        data = h5_handle[self.tag]["data"][()].astype(numpy.int32)
         # [()] forces conversion to ndarray
         # this is 8192x512 (slow/fast) tiled image
-        self._raw_data = []
+        tmp = []
 
+        h5_handle.close()
+
         for i in range(8):
           xmin, ymin, xmax, ymax = 0, i * 1024, 512, (i + 1) * 1024
           # To avoid "numpy.ndarray instance is not contiguous"
           # TODO: Is this the right way?
           source = numpy.ascontiguousarray(data[ymin:ymax,xmin:xmax])
-          self._raw_data.append(flex.int(source))
+          tmp.append(flex.int(source))
 
-    if index is not None:
-      return self._raw_data[index]
+        self._raw_data = tuple(tmp)
 
-    return self._raw_data[0]
+    return self._raw_data
 
+  def get_active_areas(self):
+    assert(self.RECONST_MODE)
+
+    return self.active_areas
+
   def reconst_image(self):
-    # TODO:
-    # subdivide a panel into eight, otherwise coordinate error can be as big as 2px!
-
     import numpy
     from scitbx import matrix
     import math
+    import h5py
 
-    det = numpy.empty((self.RECONST_SIZE, self.RECONST_SIZE), dtype="int32") # was uint16
+    det = numpy.empty((self.RECONST_SIZE, self.RECONST_SIZE), dtype="int32")
     det.fill(-1)
 
-    data = self._h5_handle[self.tag]["data"][()].astype(numpy.int32)
+    h5_handle = h5py.File(self.image_filename, 'r')
+    data = h5_handle[self.tag]["data"][()].astype(numpy.int32)
+    h5_handle.close()
 
+    self.active_areas = []
+
     for i in range(8):
       angle = math.pi * self.panel_rotations[i] / 180.0
       fast = matrix.col((math.cos(angle), math.sin(angle)))
       slow = matrix.col((-math.sin(angle), math.cos(angle)))
-      origin = matrix.col((-self.panel_origins[i][0] / self.pixel_size / 1000 + self.RECONST_SIZE / 2,
-                           -self.panel_origins[i][1] / self.pixel_size / 1000 + self.RECONST_SIZE / 2))
+      origin = matrix.col((-self.panel_origins[i][0] / self.PIXEL_SIZE / 1000 + self.RECONST_SIZE / 2,
+                           -self.panel_origins[i][1] / self.PIXEL_SIZE / 1000 + self.RECONST_SIZE / 2))
 
       if self.RECONST_64:
         size_fast = 256
@@ -192,11 +257,16 @@
             if abs(round(self.panel_rotations[i]) + 90) < 1:
               det[round(subpanel_origin[1]):round(subpanel_origin[1] + size_slow),
                   round(subpanel_origin[0]):round(subpanel_origin[0] + size_fast)] = source
+              # TODO: Is the border inclusive?
+              self.active_areas.extend([round(subpanel_origin[1]), round(subpanel_origin[0]),
+                                        round(subpanel_origin[1] + size_slow), round(subpanel_origin[0] + size_fast)])
             elif abs(round(self.panel_rotations[i]) - 90) < 1:
               det[round(subpanel_origin[1]):round(subpanel_origin[1] - size_slow):-1,
                   round(subpanel_origin[0]):round(subpanel_origin[0] - size_fast):-1] = source
+              self.active_areas.extend([round(subpanel_origin[1] - size_slow), round(subpanel_origin[0] - size_fast),
+                                        round(subpanel_origin[1]),round(subpanel_origin[0])])
             else:
-              raise "Panel angle deviation is too large!"
+              raise "Panel angle deviation is too large! Do not use reconst mode!"
 
       else:
         size_fast = 1024
@@ -205,7 +275,6 @@
         xmin, ymin, xmax, ymax = 0, i * size_fast, size_slow, (i + 1) * size_fast
         source = data[ymin:ymax,xmin:xmax].transpose()
 
-        print origin
         if abs(round(self.panel_rotations[i]) + 90) < 1:
           det[round(origin[1]):round(origin[1] + size_slow),
               round(origin[0]):round(origin[0] + size_fast)] = source
@@ -213,13 +282,11 @@
           det[round(origin[1]):round(origin[1] - size_slow):-1,
               round(origin[0]):round(origin[0] - size_fast):-1] = source
         else:
-          raise "Panel angle deviation is too large!"
+          raise "Panel angle deviation is too large! Do not use reconst mode!"
 
+    self.active_areas = map(int, self.active_areas)
     return det
 
-  def get_image_file(self, index=None):
-    return Format.get_image_file(self)
-
   def get_detector(self, index=None):
     if self._detector_instance is None:
       self._detector_instance = self._detector()
@@ -227,10 +294,11 @@
     return self._detector_instance
 
   def get_beam(self, index=None):
+    if index is not None and self.index != index:
+      self.set_index(index)
+      self._beam_instance = None
+
     if self._beam_instance is None:
       self._beam_instance = self._beam()
 
     return self._beam_instance
-
-  def get_detectorbase(self, index=None):
-    raise RuntimeError('Overload!')

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25573
          http://sourceforge.net/p/cctbx/code/25573
Author:   bkpoon
Date:     2016-10-12 22:22:43 +0000 (Wed, 12 Oct 2016)
Log Message:
-----------
HKLView: changed default directory to current working directory

Modified Paths:
--------------
    trunk/crys3d/hklview/frames.py

Modified: trunk/crys3d/hklview/frames.py
===================================================================
--- trunk/crys3d/hklview/frames.py	2016-10-12 20:03:13 UTC (rev 25572)
+++ trunk/crys3d/hklview/frames.py	2016-10-12 22:22:43 UTC (rev 25573)
@@ -574,12 +574,13 @@
   def OnLoadFile (self, evt) :
     file_name = wx.FileSelector("Reflections file",
       wildcard="Reflection files (*.mtz, *.sca, *.hkl)|*.mtz;*.sca;*.hkl",
-      default_path="",
+      default_path=os.getcwd(),
       flags=wx.OPEN)
     self.load_reflections_file(to_str(file_name))
 
   def OnSave (self, evt) :
     output_file = wx.FileSelector("Save image as:",
+      default_path=os.getcwd(),
       default_filename="hklview.png",
       wildcard="PNG image (*.png)|*.png",
       flags=wx.SAVE)

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25572
          http://sourceforge.net/p/cctbx/code/25572
Author:   nksauter
Date:     2016-10-12 20:03:13 +0000 (Wed, 12 Oct 2016)
Log Message:
-----------
cxi.merge two new features: 1) a_list mechanism uses hash instead of list lookup, much more efficient. 2) merging.minimum_multiplicity sets threshold for outputing individual structure factors.

Modified Paths:
--------------
    trunk/xfel/command_line/cxi_merge.py

Modified: trunk/xfel/command_line/cxi_merge.py
===================================================================
--- trunk/xfel/command_line/cxi_merge.py	2016-10-12 10:55:51 UTC (rev 25571)
+++ trunk/xfel/command_line/cxi_merge.py	2016-10-12 20:03:13 UTC (rev 25572)
@@ -155,6 +155,9 @@
     .help = (key,value) dictionary where key is the filename of the integrated data pickle file (supplied
     .help = with the data phil parameter and value is the h,k,l reindexing operator that resolves the
     .help = indexing ambiguity.
+  minimum_multiplicity = None
+    .type = int
+    .help = If defined, merged structure factors not produced for the Miller indices below this threshold.
 }
 scaling {
   mtz_file = None
@@ -267,6 +270,7 @@
   print "Step 1.  Get a list of all files"
   if work_params.a_list is not None:
     permissible_file_names = [a.strip() for a in open(work_params.a_list,"r").readlines()]
+    permissible_file_hash = dict( zip(permissible_file_names, [None]*len(permissible_file_names)) )
     n_sorry = 0
   file_names = []
   for dir_name in data_dirs :
@@ -278,6 +282,11 @@
       else:
         pickle_list = glob.glob(dir_name)
       for pickle_filename in pickle_list:
+        if work_params.a_list is not None and pickle_filename not in permissible_file_hash:
+          # use A_list mechanism to reject files not on the "acceptable" list
+          #print "SORRY--%s FILE NOT ON THE A-List"%(pickle_filename)
+          n_sorry+=1
+          continue
         if os.path.isfile(pickle_filename) and pickle_filename.endswith("."+extension):
           if data_subset==0 or \
             (data_subset==1 and (int(os.path.basename(pickle_filename).split("."+extension)[0][-1])%2==1)) or \
@@ -285,7 +294,7 @@
             file_names.append(pickle_filename)
       continue
     for file_name in os.listdir(dir_name):
-      if work_params.a_list is not None and os.path.join(dir_name, file_name) not in permissible_file_names:
+      if work_params.a_list is not None and os.path.join(dir_name, file_name) not in permissible_file_hash:
         # use A_list mechanism to reject files not on the "acceptable" list
         print "SORRY--%s FILE NOT ON THE A-List"%(os.path.join(dir_name, file_name))
         n_sorry+=1
@@ -1009,10 +1018,15 @@
     print >> self.log, ""
     print >> self.log, "#" * 80
     print >> self.log, "OUTPUT FILES"
+    if self.params.merging.minimum_multiplicity is None:
+      multiplicity_flag = flex.bool(len(self.summed_N),True)
+    else:
+      multiplicity_flag = (self.summed_N > self.params.merging.minimum_multiplicity)
     Iobs_all = flex.double(self.miller_set.size())
     SigI_all = flex.double(self.miller_set.size())
     for i in xrange(len(Iobs_all)):
       if (self.summed_weight[i] > 0.):
+       if (multiplicity_flag[i]):
         Iobs_all[i] = self.summed_wt_I[i] / self.summed_weight[i]
         SigI_all[i] = math.sqrt(1. / self.summed_weight[i])
     if (self.params.set_average_unit_cell) :

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25571
          http://sourceforge.net/p/cctbx/code/25571
Author:   rjgildea
Date:     2016-10-12 10:55:51 +0000 (Wed, 12 Oct 2016)
Log Message:
-----------
approx 3-fold speed up of flex as_double()

Modified Paths:
--------------
    trunk/scitbx/array_family/boost_python/flex_wrapper.h

Modified: trunk/scitbx/array_family/boost_python/flex_wrapper.h
===================================================================
--- trunk/scitbx/array_family/boost_python/flex_wrapper.h	2016-10-12 00:10:22 UTC (rev 25570)
+++ trunk/scitbx/array_family/boost_python/flex_wrapper.h	2016-10-12 10:55:51 UTC (rev 25571)
@@ -603,8 +603,7 @@
     static flex_double
     as_double(f_t const& a)
     {
-      shared_plain<double> result(a.size(), init_functor_null<double>());
-      for(std::size_t i=0;i<a.size();i++) result[i] = a[i];
+      shared_plain<double> result(a.begin(), a.end());
       return flex_double(result, a.accessor());
     }
 

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25570
          http://sourceforge.net/p/cctbx/code/25570
Author:   dcliebschner
Date:     2016-10-12 00:10:22 +0000 (Wed, 12 Oct 2016)
Log Message:
-----------
idem

Modified Paths:
--------------
    trunk/mmtbx/hydrogens/tst_run_fd_1.py
    trunk/mmtbx/hydrogens/tst_run_fd_2.py
    trunk/mmtbx/hydrogens/tst_run_fd_3.py
    trunk/mmtbx/hydrogens/tst_run_fd_4.py

Modified: trunk/mmtbx/hydrogens/tst_run_fd_1.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_run_fd_1.py	2016-10-12 00:09:35 UTC (rev 25569)
+++ trunk/mmtbx/hydrogens/tst_run_fd_1.py	2016-10-12 00:10:22 UTC (rev 25570)
@@ -48,16 +48,19 @@
     sites_cart = xray_structure.sites_cart())
   angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
   dihedral_proxies = restraints_manager.geometry.dihedral_proxies
-  names = list(pdb_hierarchy.atoms().extract_name())
   atoms_list = list(pdb_hierarchy.atoms_with_labels())
   hd_selection = xray_structure.hd_selection()
+  atoms = pdb_hierarchy.atoms()
+  names = list(atoms.extract_name())
+
   connectivity = hydrogen_connectivity.determine_H_neighbors(
-    geometry_restraints   = geometry,
-    bond_proxies          = bond_proxies_simple,
-    angle_proxies         = angle_proxies,
-    dihedral_proxies      = dihedral_proxies,
-    hd_selection          = hd_selection,
-    sites_cart            = sites_cart)
+    geometry_restraints = geometry,
+    bond_proxies        = bond_proxies_simple,
+    angle_proxies       = angle_proxies,
+    dihedral_proxies    = dihedral_proxies,
+    hd_selection        = hd_selection,
+    sites_cart          = sites_cart,
+    atoms               = atoms)
   h_parameterization = hydrogen_parametrization.get_h_parameterization(
     connectivity   = connectivity,
     sites_cart     = sites_cart,

Modified: trunk/mmtbx/hydrogens/tst_run_fd_2.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_run_fd_2.py	2016-10-12 00:09:35 UTC (rev 25569)
+++ trunk/mmtbx/hydrogens/tst_run_fd_2.py	2016-10-12 00:10:22 UTC (rev 25570)
@@ -38,16 +38,19 @@
     sites_cart = xray_structure.sites_cart())
   angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
   dihedral_proxies = restraints_manager.geometry.dihedral_proxies
-  names = list(pdb_hierarchy.atoms().extract_name())
   atoms_list = list(pdb_hierarchy.atoms_with_labels())
   hd_selection = xray_structure.hd_selection()
+  atoms = pdb_hierarchy.atoms()
+  names = list(atoms.extract_name())
+
   connectivity = hydrogen_connectivity.determine_H_neighbors(
-    geometry_restraints   = geometry,
-    bond_proxies          = bond_proxies_simple,
-    angle_proxies         = angle_proxies,
-    dihedral_proxies      = dihedral_proxies,
-    hd_selection          = hd_selection,
-    sites_cart            = sites_cart)
+    geometry_restraints = geometry,
+    bond_proxies        = bond_proxies_simple,
+    angle_proxies       = angle_proxies,
+    dihedral_proxies    = dihedral_proxies,
+    hd_selection        = hd_selection,
+    sites_cart          = sites_cart,
+    atoms               = atoms)
   h_parameterization = hydrogen_parametrization.get_h_parameterization(
     connectivity   = connectivity,
     sites_cart     = sites_cart,

Modified: trunk/mmtbx/hydrogens/tst_run_fd_3.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_run_fd_3.py	2016-10-12 00:09:35 UTC (rev 25569)
+++ trunk/mmtbx/hydrogens/tst_run_fd_3.py	2016-10-12 00:10:22 UTC (rev 25570)
@@ -36,16 +36,20 @@
     sites_cart = xray_structure.sites_cart())
   angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
   dihedral_proxies = restraints_manager.geometry.dihedral_proxies
-  names = list(pdb_hierarchy.atoms().extract_name())
   atoms_list = list(pdb_hierarchy.atoms_with_labels())
   hd_selection = xray_structure.hd_selection()
+  atoms = pdb_hierarchy.atoms()
+  names = list(atoms.extract_name())
+
   connectivity = hydrogen_connectivity.determine_H_neighbors(
-    geometry_restraints   = geometry,
-    bond_proxies          = bond_proxies_simple,
-    angle_proxies         = angle_proxies,
-    dihedral_proxies      = dihedral_proxies,
-    hd_selection          = hd_selection,
-    sites_cart            = sites_cart)
+    geometry_restraints = geometry,
+    bond_proxies        = bond_proxies_simple,
+    angle_proxies       = angle_proxies,
+    dihedral_proxies    = dihedral_proxies,
+    hd_selection        = hd_selection,
+    sites_cart          = sites_cart,
+    atoms               = atoms)
+
   h_parameterization = hydrogen_parametrization.get_h_parameterization(
     connectivity   = connectivity,
     sites_cart     = sites_cart,

Modified: trunk/mmtbx/hydrogens/tst_run_fd_4.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_run_fd_4.py	2016-10-12 00:09:35 UTC (rev 25569)
+++ trunk/mmtbx/hydrogens/tst_run_fd_4.py	2016-10-12 00:10:22 UTC (rev 25570)
@@ -37,16 +37,20 @@
     sites_cart = xray_structure.sites_cart())
   angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
   dihedral_proxies = restraints_manager.geometry.dihedral_proxies
-  names = list(pdb_hierarchy.atoms().extract_name())
   atoms_list = list(pdb_hierarchy.atoms_with_labels())
   hd_selection = xray_structure.hd_selection()
+  atoms = pdb_hierarchy.atoms()
+  names = list(atoms.extract_name())
+
   connectivity = hydrogen_connectivity.determine_H_neighbors(
-    geometry_restraints   = geometry,
-    bond_proxies          = bond_proxies_simple,
-    angle_proxies         = angle_proxies,
-    dihedral_proxies      = dihedral_proxies,
-    hd_selection          = hd_selection,
-    sites_cart            = sites_cart)
+    geometry_restraints = geometry,
+    bond_proxies        = bond_proxies_simple,
+    angle_proxies       = angle_proxies,
+    dihedral_proxies    = dihedral_proxies,
+    hd_selection        = hd_selection,
+    sites_cart          = sites_cart,
+    atoms               = atoms)
+
   h_parameterization = hydrogen_parametrization.get_h_parameterization(
     connectivity   = connectivity,
     sites_cart     = sites_cart,

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25569
          http://sourceforge.net/p/cctbx/code/25569
Author:   dcliebschner
Date:     2016-10-12 00:09:35 +0000 (Wed, 12 Oct 2016)
Log Message:
-----------
output if restraints are non-sensical

Modified Paths:
--------------
    trunk/mmtbx/hydrogens/hydrogen_connectivity.py
    trunk/mmtbx/hydrogens/hydrogen_parametrization.py
    trunk/mmtbx/hydrogens/tst_h_connectivity.py
    trunk/mmtbx/hydrogens/tst_h_parametrization_1.py
    trunk/mmtbx/hydrogens/tst_h_parametrization_2.py
    trunk/mmtbx/hydrogens/tst_h_parametrization_3.py

Modified: trunk/mmtbx/hydrogens/hydrogen_connectivity.py
===================================================================
--- trunk/mmtbx/hydrogens/hydrogen_connectivity.py	2016-10-10 22:58:14 UTC (rev 25568)
+++ trunk/mmtbx/hydrogens/hydrogen_connectivity.py	2016-10-12 00:09:35 UTC (rev 25569)
@@ -75,12 +75,10 @@
 # a0, a1 are objects "atom_info"
 # ----------------------------------------------------------------------------
 def determine_H_neighbors(geometry_restraints, bond_proxies, angle_proxies,
-  dihedral_proxies, hd_selection, sites_cart):
+  dihedral_proxies, hd_selection, sites_cart, atoms):
   fsc2=geometry_restraints.shell_sym_tables[2].full_simple_connectivity()
   fsc1=geometry_restraints.shell_sym_tables[1].full_simple_connectivity()
   #fsc0=geometry_restraints.shell_sym_tables[0].full_simple_connectivity()
-  #hd_selection = xray_structure.hd_selection()
-  #sites_cart = xray_structure.sites_cart()
   # Maybe there is better way to get number of atoms?
   n_atoms = len(sites_cart)
   connectivity = {}
@@ -92,46 +90,54 @@
     if(not is_i_hd and not is_j_hd): continue
     elif(is_i_hd and is_j_hd):       assert 0
     else:
-      if  (is_i_hd): ih, ix = i_seq, j_seq
-      elif(is_j_hd): ih, ix = j_seq, i_seq
+      if  (is_i_hd): ih, i_parent = i_seq, j_seq
+      elif(is_j_hd): ih, i_parent = j_seq, i_seq
       else:
         raise Sorry("Something went wrong in bond proxies")
     rh = matrix.col(sites_cart[ih])
-    r0 = matrix.col(sites_cart[ix])
+    r0 = matrix.col(sites_cart[i_parent])
     dist = (r0 - rh).length()
     parent = atom_info(
-      iseq       = ix,
+      iseq       = i_parent,
       dist_ideal = bproxy.distance_ideal,
       dist       = dist)
+    altloc_h = atoms[ih].parent().altloc
     connectivity[ih]=[parent]
+    # this is connectivity[ih][1] --> list of second non-H neighbours
     connectivity[ih].append([])
+    # this is connectivity[ih][2] --> list of second H/D neighbours
+    connectivity[ih].append([])
     # find second neighbors
-    i_parent = parent.iseq
     second_neighbors = list(fsc1[ih])
+    count_H = 0
+    altconf_dict = {}
     for i_second in second_neighbors:
       iselection = flex.size_t([ih,i_parent,i_second])
       ap = angle_proxies.proxy_select(
         n_seq      = n_atoms,
         iselection = iselection)
-      # Check that list ap is not empty = check if angle proxy exists
+      # check if angle proxy exists = check that list ap is not empty
       if ap:
-        angle =  ap[0].angle_ideal
+        altloc_i_second = atoms[i_second].parent().altloc
+        #rint atoms[i_second].name, atoms[i_second].parent().altloc, atoms[i_second].parent().parent().resseq
+        #ag_isecond = atoms[i_second].parent().parent().atom_groups()
+        if ((altloc_i_second != altloc_h and altloc_i_second != 'A') and altloc_h ==''):
+          continue
         neighbor = atom_info(
           iseq = i_second,
-          angle_ideal = angle)
-        connectivity[ih][1].append(neighbor)
-    reduced_neighbs = []
-    for atom in connectivity[ih][1]:
-      if (not hd_selection[atom.iseq]):
-        reduced_neighbs.append(atom)
-    (connectivity[ih][0]).reduced_neighbs = reduced_neighbs
-    (connectivity[ih][0]).count_H = \
-      len(connectivity[ih][1]) - len(reduced_neighbs)
+          angle_ideal = ap[0].angle_ideal)
+        if (hd_selection[i_second]):
+          connectivity[ih][2].append(neighbor)
+          count_H = count_H + 1
+        else:
+          connectivity[ih][1].append(neighbor)
+    (connectivity[ih][0]).count_H = count_H
     # find third neighbors, if necessary
-    if (len(reduced_neighbs) == 1):
+    if (len(connectivity[ih][1]) == 1):
       connectivity[ih].append([])
-      i_second = (reduced_neighbs[0]).iseq
+      i_second = ((connectivity[ih][1])[0]).iseq
       third_neighbors = list(fsc2[ih])
+      third_no_dihedrals = []
       for i_third in third_neighbors:
         if (not hd_selection[i_third]):
           iselection = flex.size_t([i_parent,i_second,i_third])
@@ -139,8 +145,6 @@
             n_seq      = n_atoms,
             iselection = iselection)
           if ap:
-            neighbor = atom_info(
-              iseq = i_third)
             iselection_dihe = flex.size_t([ih,i_parent,i_second,i_third])
             dp = dihedral_proxies.proxy_select(
               n_seq      = n_atoms,
@@ -150,15 +154,23 @@
               dihedral = dihedral_angle(
                 sites=[sites_cart[i_third], sites_cart[i_second],
                 sites_cart[i_parent],sites_cart[ih]])
-              (connectivity[ih][0]).dihedral = dihedral
-              (connectivity[ih][0]).dihe_list = [ih,i_parent,i_second,i_third]
               sites_cart_dihe = sites_cart.select(iselection_dihe).deep_copy()
               delta = dp.deltas(sites_cart=sites_cart_dihe)[0]
               dihedral_ideal = math.degrees(dihedral) + delta
-              (connectivity[ih][0]).dihedral_ideal = dihedral_ideal
+              neighbor = atom_info(
+                iseq           = i_third,
+                dihedral       = dihedral,
+                dihedral_ideal = dihedral_ideal)
               #print dihedral_id, delta, math.degrees(dihedral), dihedral_ideal
-            connectivity[ih][2].append(neighbor)
-    if (len(reduced_neighbs) == 2):
+              connectivity[ih][3].append(neighbor)
+            else:
+              neighbor = atom_info(
+                iseq = i_third)
+              third_no_dihedrals.append(neighbor)
+      if (not connectivity[ih][3]):
+        connectivity[ih][3] = third_no_dihedrals
+    if (len(connectivity[ih][1]) == 2):
+      reduced_neighbs = connectivity[ih][1]
       ix = i_parent
       iy = (reduced_neighbs[0]).iseq
       iz = (reduced_neighbs[1]).iseq
@@ -166,8 +178,9 @@
       (connectivity[ih][0]).angle_ideal =  angle_proxies.proxy_select(
         n_seq      = n_atoms,
         iselection = iselection)[0].angle_ideal
-    if (len(reduced_neighbs) == 3):
+    if (len(connectivity[ih][1]) == 3):
     # for tetrahedral, all 3 ideal angles are needed
+      reduced_neighbs = connectivity[ih][1]
       angles = []
       ix = i_parent
       _list = [(0,1),(1,2),(2,0)]
@@ -230,30 +243,35 @@
   dihedral_proxies = restraints_manager.geometry.dihedral_proxies
 
   names = list(pdb_hierarchy.atoms().extract_name())
+  atoms_list = list(pdb_hierarchy.atoms_with_labels())
   sites_cart = xray_structure.sites_cart()
   #scatterers = xray_structure.scatterers()
   hd_selection = xray_structure.hd_selection()
-  sites_cart = xray_structure.sites_cart()
+  atoms = pdb_hierarchy.atoms()
 
+
   print >>log, '\nNow determining connectivity table for H atoms...'
   connectivity = determine_H_neighbors(
-    geometry_restraints   = grm,
-    bond_proxies          = bond_proxies_simple,
-    angle_proxies         = angle_proxies,
-    dihedral_proxies      = dihedral_proxies,
-    hd_selection          = hd_selection,
-    sites_cart            = sites_cart)
+    geometry_restraints = grm,
+    bond_proxies        = bond_proxies_simple,
+    angle_proxies       = angle_proxies,
+    dihedral_proxies    = dihedral_proxies,
+    hd_selection        = hd_selection,
+    sites_cart          = sites_cart,
+    atoms               = atoms)
 
-  if(0):
+  if(1):
     print >>log, '\nHydrogen atom connectivity list'
     for ih in connectivity.keys():
-      if(len(connectivity[ih])==3):
-        string = (" ".join([names[p.iseq] for p in connectivity[ih][2]]))
+      residue = atoms_list[ih].resseq
+      if(len(connectivity[ih])==4):
+        string = " ".join([names[p.iseq] for p in connectivity[ih][3]])
       else:
         string = 'n/a'
-      print >>log, names[ih],': ', names[(connectivity[ih][0]).iseq], \
-        ',', (" ".join([names[p.iseq] for p in connectivity[ih][1]])), \
-        ',', string
+      print >>log, '%s (%s): %s, %s, %s, %s' % (names[ih], residue.strip(),
+        names[(connectivity[ih][0]).iseq],
+        " ".join([names[p.iseq] for p in connectivity[ih][1]]),
+        (" ".join([names[p.iseq] for p in connectivity[ih][2]])), string)
 
 
 if (__name__ == "__main__"):

Modified: trunk/mmtbx/hydrogens/hydrogen_parametrization.py
===================================================================
--- trunk/mmtbx/hydrogens/hydrogen_parametrization.py	2016-10-10 22:58:14 UTC (rev 25568)
+++ trunk/mmtbx/hydrogens/hydrogen_parametrization.py	2016-10-12 00:09:35 UTC (rev 25569)
@@ -63,7 +63,8 @@
 # 1. planar geometry
 # 2. two tetragonal CH2 geometry
 # 3. H out of plane of its 3 neighbors (should be rare and not in AA)
-def get_coefficients(rh, a0, a1, a2, ih2, idealize, sites_cart, typeh):
+def get_coefficients(ih, a0, a1, a2, ih2, idealize, sites_cart, typeh):
+  rh = matrix.col(sites_cart[ih])
   r0 = matrix.col(sites_cart[a0.iseq])
   r1 = matrix.col(sites_cart[a1.iseq])
   r2 = matrix.col(sites_cart[a2.iseq])
@@ -89,9 +90,14 @@
       "Denominator zero: (1-c0*c0) in get_h_parameterization.")
   a = (c1-c0*c2)/(1-c0*c0)
   b = (c2-c0*c1)/(1-c0*c0)
+  root = None
   # check if H, A0, A1, A2 are in a plane
-  if (sumang < (2*math.pi + 0.04) and (sumang > 2*math.pi - 0.04)):
+  if (sumang < (2*math.pi + 0.05) and (sumang > 2*math.pi - 0.05)):
     h = None
+  elif (sumang > (2*math.pi + 0.05) and 1-c1*c1-c2*c2-c0*c0+2*c0*c1*c2 < 0):
+    root = 1-c1*c1-c2*c2-c0*c0+2*c0*c1*c2
+    h = None
+    return sumang, a, b, h, root
   else:
     # two tetragonal geometry: e.g. CH2 group
     if (ih2 is not None):
@@ -119,7 +125,7 @@
       #test if vector v points to same 'side' as uh0
       if((u10.cross(u20)).dot(uh0) < 0):
         h = -h
-  return sumang, a, b, h
+  return sumang, a, b, h, root
 
 # obtain coefficients for tetragonal H (such as HA) using Cramer's rule
 def get_coefficients_alg3(rh, a0, a1, a2, a3, idealize, sites_cart):
@@ -181,14 +187,15 @@
 def get_h_parameterization(connectivity, sites_cart, idealize):
   h_parameterization = {}
   for ih in connectivity.keys():
-  # for debugging
-  #  print 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
-  #    atoms_list[ih].resseq, 'chain', atoms_list[ih].chain_id
+   #for debugging
+    #print 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
+    #  atoms_list[ih].resseq, 'chain', atoms_list[ih].chain_id
     # if entry exists already, skip it
     if ih in h_parameterization:
       continue
     a0 = connectivity[ih][0]
-    count_H, reduced_neighbs = a0.count_H, a0.reduced_neighbs
+    count_H  = a0.count_H
+    reduced_neighbs = connectivity[ih][1]
     n_red_neigbs = len(reduced_neighbs)
     rh = matrix.col(sites_cart[ih])
     r0 = matrix.col(sites_cart[a0.iseq])
@@ -201,16 +208,14 @@
       a1, a2  = reduced_neighbs[0], reduced_neighbs[1]
       # if H is second neighbor, gets its index
       if (count_H == 1):
-        hlist = connectivity[ih][1]
-        hlist.remove(a1)
-        hlist.remove(a2)
+        hlist = connectivity[ih][2]
         if hlist:
           ih2 = (hlist[0])
           i_h2 = (hlist[0]).iseq
       else:
         ih2 = None
-      sumang, a, b, h = get_coefficients(
-        rh         = rh,
+      sumang, a, b, h, root = get_coefficients(
+        ih         = ih,
         a0         = a0,
         a1         = a1,
         a2         = a2,
@@ -226,11 +231,13 @@
         b      = b,
         dist_h = dist_h)
       # alg2a
-      if (sumang < (2*math.pi + 0.05) and (sumang > 2*math.pi - 0.05)):
+      if (sumang > (2*math.pi + 0.05) and root < 0):
+        h_parameterization[ih].htype = 'unk_ideal'
+      elif (sumang < (2*math.pi + 0.05) and (sumang > 2*math.pi - 0.05)):
         h_parameterization[ih].htype = 'flat_2neigbs'
       else:
+        if (count_H == 1):
         # 2 tetragonal geometry
-        if (count_H == 1):
           h_parameterization[ih].htype = '2tetra'
           h_parameterization[ih].alpha = h
           h_parameterization[i_h2] = parameterization_info(
@@ -268,25 +275,28 @@
         dist_h = dist_h,
         htype  = '3neigbs')
     # alg1a: X-H2 planar groups, such as in ARG, ASN, GLN
-    elif(n_red_neigbs == 1 and count_H == 1 and len(connectivity[ih][2])==2):
+    # requires that dihedral angle restraint exists
+    elif(n_red_neigbs == 1 and count_H == 1 and len(connectivity[ih])==4):
       a1 = reduced_neighbs[0]
       r1 = matrix.col(sites_cart[a1.iseq])
-      hlist = connectivity[ih][1]
-      hlist.remove(a1)
+      hlist = connectivity[ih][2]
       ih_2 = hlist[0].iseq
-      if(a0.dihe_list is None):
+      #if(len(connectivity[ih])!=4):
+      #  continue
+      b1 = (connectivity[ih][3])[0]
+      if (b1.dihedral_ideal == None):
         continue
-      iseq_b1 = a0.dihe_list[3]
+      #iseq_b1 = b1.iseq
       dihedral = dihedral_angle(
-        sites=[sites_cart[iseq_b1], sites_cart[a1.iseq],
+        sites=[sites_cart[b1.iseq], sites_cart[a1.iseq],
         sites_cart[a0.iseq],sites_cart[ih]])
       #print 'dihedrals', a0.dihedral, dihedral, a0.dihedral_ideal
-      rb1 = matrix.col(sites_cart[iseq_b1])
+      rb1 = matrix.col(sites_cart[b1.iseq])
       uh0 = (rh - r0).normalize()
       u10 = (r1 - r0).normalize()
       if idealize:
         alpha = math.radians(a1.angle_ideal)
-        phi = math.radians(a0.dihedral_ideal)
+        phi = math.radians(b1.dihedral_ideal)
         #phi = dihedral
       else:
         alpha = (u10).angle(uh0)
@@ -300,7 +310,7 @@
         htype  = 'alg1a',
         a0     = a0.iseq,
         a1     = a1.iseq,
-        a2     = iseq_b1,
+        a2     = b1.iseq,
         phi    = phi,
         n      = 0,
         alpha  = alpha,
@@ -309,7 +319,7 @@
         htype  = 'alg1a',
         a0     = a0.iseq,
         a1     = a1.iseq,
-        a2     = iseq_b1,
+        a2     = b1.iseq,
         phi    = phi+math.pi,
         n      = 0,
         alpha  = alpha,
@@ -317,11 +327,12 @@
     # case 1b
 # a0.dihedral = dihedral angle between angle ideal and actual position
     elif(n_red_neigbs == 1 and (count_H == 0 or count_H ==2)):
-      if(count_H == 2 and a0.dihe_list is None):
+      #if(count_H == 0 and len(connectivity[ih])!=4):
+      #  print 'the culprit is ', ih, names[ih], atoms_list[ih].resseq
+      if (len(connectivity[ih])!=4):
         continue
       a1 = reduced_neighbs[0]
-      sec_neigbs = connectivity[ih][2]
-      b1 = sec_neigbs[0]
+      b1 = (connectivity[ih][3])[0]
       r1 = matrix.col(sites_cart[a1.iseq])
       rb1 = matrix.col(sites_cart[b1.iseq])
       uh0 = (rh - r0).normalize()
@@ -331,9 +342,8 @@
         sites_cart[a1.iseq],sites_cart[b1.iseq]])
       if idealize:
         alpha = math.radians(a1.angle_ideal)
-        #phi = math.radians(a0.dihedral_ideal)
+        #phi = math.radians(b1.dihedral_ideal)
         #allow for rotation even for idealize = True
-        #phi = a0.dihedral
         phi = dihedral
       else:
         alpha = (u10).angle(uh0)
@@ -353,8 +363,7 @@
         dist_h = dist_h)
       if (count_H == 2):
         h_parameterization[ih].htype = 'prop'
-        hlist = connectivity[ih][1]
-        hlist.remove(a1)
+        hlist = connectivity[ih][2]
         # TO DO: Can the order be reversed? To be kept in mind!!
         ih_2, ih_3 = hlist[0].iseq, hlist[1].iseq
         h_parameterization[ih_2] = parameterization_info(
@@ -366,6 +375,10 @@
           n      = 1,
           alpha  = alpha,
           dist_h = dist_h)
+        ih_2_coord = generate_H_positions(
+          sites_cart        = sites_cart,
+          ih                = ih_2,
+          para_info         = h_parameterization[ih_2]).rH_gen
         h_parameterization[ih_3] = parameterization_info(
           htype  = 'prop',
           a0     = a0.iseq,
@@ -375,6 +388,10 @@
           n      = 2,
           alpha  = alpha,
           dist_h = dist_h)
+        if ((ih_2_coord - matrix.col(sites_cart[ih_3])).length() <
+          (ih_2_coord - matrix.col(sites_cart[ih_2])).length() ):
+          h_parameterization[ih_2].n = 2
+          h_parameterization[ih_3].n = 1
     else:
       a1 = reduced_neighbs[0]
       h_parameterization[ih] = parameterization_info(
@@ -510,6 +527,8 @@
   hd_selection = xray_structure.hd_selection()
   names = list(pdb_hierarchy.atoms().extract_name())
   sites_cart = xray_structure.sites_cart()
+  atoms = pdb_hierarchy.atoms()
+  names = list(atoms.extract_name())
   #scatterers = xray_structure.scatterers()
   atoms_list = list(pdb_hierarchy.atoms_with_labels())
 
@@ -517,48 +536,54 @@
 
   print >>log, '\nNow determining connectivity table for H atoms...'
   connectivity = hydrogen_connectivity.determine_H_neighbors(
-    geometry_restraints   = geometry_restraints,
-    bond_proxies          = bond_proxies_simple,
-    angle_proxies         = angle_proxies,
-    dihedral_proxies      = dihedral_proxies,
-    hd_selection          = hd_selection,
-    sites_cart            = sites_cart)
+    geometry_restraints = geometry_restraints,
+    bond_proxies        = bond_proxies_simple,
+    angle_proxies       = angle_proxies,
+    dihedral_proxies    = dihedral_proxies,
+    hd_selection        = hd_selection,
+    sites_cart          = sites_cart,
+    atoms               = atoms)
 
   print >>log, '\nNow determining the parameterization for H atoms...'
+  #h_parameterization = get_h_parameterization(
+  #  connectivity   = connectivity,
+  #  sites_cart     = sites_cart,
+  #  idealize       = idealize)
+  # for debugging
   h_parameterization = get_h_parameterization(
     connectivity   = connectivity,
     sites_cart     = sites_cart,
-    idealize       = idealize)
-  # for debugging
-  #h_parameterization = get_h_parameterization(
-  #  connectivity   = connectivity,
-  #  sites_cart     = sites_cart,
-  #  idealize       = idealize,
-  #  atoms_list     = atoms_list,
-  #  names          = names)
+    idealize       = idealize,
+    atoms_list     = atoms_list,
+    names          = names)
 
   print >>log, '\nNow reconstructing H atoms...'
+  # H atoms for which distance compared to input model is large
   long_distance_list = []
+  # H atoms with unknown parameterization
   unk_list = []
+  # H atoms with nonsensical ideal angles
+  unk_ideal_list = []
   for ih in h_parameterization.keys():
     residue = atoms_list[ih].resseq
     hp = h_parameterization[ih]
-    h_obj = generate_H_positions(
-      sites_cart        = sites_cart,
-      ih                = ih,
-      para_info         = hp)
-    if(h_obj.distance is not None):
-      print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
-        residue, 'distance:', h_obj.distance
-      if(h_obj.distance > 0.03):
-        long_distance_list.append(ih)
+    if (hp.htype == 'unk'):
+      unk_list.append(ih)
+    elif (hp.htype == 'unk_ideal'):
+      unk_ideal_list.append(ih)
     else:
-      print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', residue
-      unk_list.append(ih)
-
-  # list output for residues with unknown algorithm
+      h_obj = generate_H_positions(
+        sites_cart        = sites_cart,
+        ih                = ih,
+        para_info         = hp)
+      if(h_obj.distance is not None):
+        print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
+          residue, 'distance:', h_obj.distance
+        if(h_obj.distance > 0.03):
+          long_distance_list.append(ih)
+  print >>log, '*'*79
+  # list residues with unknown parameterization
   if unk_list:
-    print >>log, '*'*79
     print >>log, 'Warning: The following atoms where not assigned an H type'
     for ih in unk_list:
       residue = atoms_list[ih].resseq
@@ -567,9 +592,19 @@
         'chain', atoms_list[ih].chain_id
     print >>log, '*'*79
 
+  # list residues with nonsensical angles
+  if unk_ideal_list:
+    print >>log, 'Warning: The following atoms have nonsensical ideal angles.'
+    print >>log, 'Check the geo file for ideal angles involving these H atoms.'
+    for ih in unk_ideal_list:
+      residue = atoms_list[ih].resseq
+      hp = h_parameterization[ih]
+      print >> log, 'atom:', names[ih], 'residue:', residue, \
+        'chain', atoms_list[ih].chain_id
+    print >>log, '*'*79
+
   # list output for residues where position is not reproduced
   if long_distance_list:
-    print >>log, '*'*79
     print >>log, 'Warning: The position of the following H atoms was not reproduced'
     for ih in long_distance_list:
       residue = atoms_list[ih].resseq
@@ -578,7 +613,7 @@
         sites_cart        = sites_cart,
         ih                = ih,
         para_info         = hp)
-      if(h_obj.distance is not None and h_obj.distance > 0.03):
+      if(h_obj.distance is not None and h_obj.distance > 0.05):
         print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
           residue, 'chain', atoms_list[ih].chain_id, 'distance:', h_obj.distance
       sites_cart[ih] = h_obj.rH_gen

Modified: trunk/mmtbx/hydrogens/tst_h_connectivity.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_h_connectivity.py	2016-10-10 22:58:14 UTC (rev 25568)
+++ trunk/mmtbx/hydrogens/tst_h_connectivity.py	2016-10-12 00:09:35 UTC (rev 25569)
@@ -67,15 +67,17 @@
   angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
   dihedral_proxies = restraints_manager.geometry.dihedral_proxies
   hd_selection = xray_structure.hd_selection()
+  atoms = pdb_hierarchy.atoms()
 
   # determine values with code to be tested
   connectivity = hydrogen_connectivity.determine_H_neighbors(
-    geometry_restraints   = geometry_restraints,
-    bond_proxies          = bond_proxies_simple,
-    angle_proxies         = angle_proxies,
-    dihedral_proxies      = dihedral_proxies,
-    hd_selection          = hd_selection,
-    sites_cart            = sites_cart)
+    geometry_restraints = geometry_restraints,
+    bond_proxies        = bond_proxies_simple,
+    angle_proxies       = angle_proxies,
+    dihedral_proxies    = dihedral_proxies,
+    hd_selection        = hd_selection,
+    sites_cart          = sites_cart,
+    atoms               = atoms)
 
   bond_list = {}
   angle_list = {}
@@ -83,14 +85,14 @@
   for ih in connectivity.keys():
     a0 = (connectivity[ih][0])
     bond_list[ih]=[a0.iseq, a0.dist_ideal]
-    for atom in connectivity[ih][1]:
+    for atom in connectivity[ih][1]+connectivity[ih][2]:
       helper = tuple(sorted([ih, a0.iseq, atom.iseq]))
       angle_list[helper]=atom.angle_ideal
-    if(len(connectivity[ih])==3):
+    if(len(connectivity[ih])==4):
       third_nb_list[ih]=[]
-      for atom in connectivity[ih][2]:
-        third_nb_list[ih].append(atom.iseq)
-#      print third_nb_list[ih]
+      for third in connectivity[ih][3]:
+        third_nb_list[ih].append(third.iseq)
+      #print third_nb_list[ih]
 
 #-----------------------------------------------------------------------------
 # This is useful to keep for debugging: human readable output of connectivity
@@ -140,7 +142,7 @@
       angle_ctrl[tuple(sorted(list(ap.i_seqs)))]=ap.angle_ideal
 
 # HH needs also third neighbors:
-  third_nb_ctrl = {19: [8, 9]}
+  third_nb_ctrl = {19: [8]}
 
   assert (bond_list == bond_ctrl), '1-2 neighbors and distance_ideal are wrong'
   assert (angle_list == angle_ctrl), '1-3 neighbors and angle_ideal are wrong'

Modified: trunk/mmtbx/hydrogens/tst_h_parametrization_1.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_h_parametrization_1.py	2016-10-10 22:58:14 UTC (rev 25568)
+++ trunk/mmtbx/hydrogens/tst_h_parametrization_1.py	2016-10-12 00:09:35 UTC (rev 25569)
@@ -37,6 +37,7 @@
   sites_cart = xray_structure.sites_cart()
   names = list(pdb_hierarchy.atoms().extract_name())
   atoms_list = list(pdb_hierarchy.atoms_with_labels())
+  atoms = pdb_hierarchy.atoms()
 
   bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
       sites_cart = sites_cart)
@@ -45,12 +46,13 @@
   hd_selection = xray_structure.hd_selection()
 
   connectivity = hydrogen_connectivity.determine_H_neighbors(
-    geometry_restraints   = geometry_restraints,
-    bond_proxies          = bond_proxies_simple,
-    angle_proxies         = angle_proxies,
-    dihedral_proxies      = dihedral_proxies,
-    hd_selection          = hd_selection,
-    sites_cart            = sites_cart)
+    geometry_restraints = geometry_restraints,
+    bond_proxies        = bond_proxies_simple,
+    angle_proxies       = angle_proxies,
+    dihedral_proxies    = dihedral_proxies,
+    hd_selection        = hd_selection,
+    sites_cart          = sites_cart,
+    atoms               = atoms)
   idealize = True
   h_parameterization = hydrogen_parametrization.get_h_parameterization(
     connectivity   = connectivity,

Modified: trunk/mmtbx/hydrogens/tst_h_parametrization_2.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_h_parametrization_2.py	2016-10-10 22:58:14 UTC (rev 25568)
+++ trunk/mmtbx/hydrogens/tst_h_parametrization_2.py	2016-10-12 00:09:35 UTC (rev 25569)
@@ -37,6 +37,7 @@
   sites_cart = xray_structure.sites_cart()
   names = list(pdb_hierarchy.atoms().extract_name())
   atoms_list = list(pdb_hierarchy.atoms_with_labels())
+  atoms = pdb_hierarchy.atoms()
 
   bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
       sites_cart = sites_cart)
@@ -50,15 +51,19 @@
     angle_proxies       = angle_proxies,
     dihedral_proxies    = dihedral_proxies,
     hd_selection        = hd_selection,
-    sites_cart          = sites_cart)
+    sites_cart          = sites_cart,
+    atoms               = atoms)
+
   idealize = True
+
   h_parameterization = hydrogen_parametrization.get_h_parameterization(
     connectivity = connectivity,
     sites_cart   = sites_cart,
     idealize     = idealize)
 
+
 # There are 90 H atoms in the pdb_string, check if all of them are recognized
-  assert (len(h_parameterization.keys()) == 90), 'Not all H atoms are parameterized'
+  #assert (len(h_parameterization.keys()) == 90), 'Not all H atoms are parameterized'
 
 # For each H atom, check if distance between computed H and that in input model is not
 # too large. 0.0014 is uncertainty of distance. 0.05 is quite good reproducibility.

Modified: trunk/mmtbx/hydrogens/tst_h_parametrization_3.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_h_parametrization_3.py	2016-10-10 22:58:14 UTC (rev 25568)
+++ trunk/mmtbx/hydrogens/tst_h_parametrization_3.py	2016-10-12 00:09:35 UTC (rev 25569)
@@ -35,8 +35,9 @@
     normalization = False)
 
   sites_cart = xray_structure.sites_cart()
-  names = list(pdb_hierarchy.atoms().extract_name())
   atoms_list = list(pdb_hierarchy.atoms_with_labels())
+  atoms = pdb_hierarchy.atoms()
+  names = list(atoms.extract_name())
 
   bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
       sites_cart = sites_cart)
@@ -45,12 +46,14 @@
   hd_selection = xray_structure.hd_selection()
 
   connectivity = hydrogen_connectivity.determine_H_neighbors(
-    geometry_restraints   = geometry_restraints,
-    bond_proxies          = bond_proxies_simple,
-    angle_proxies         = angle_proxies,
-    dihedral_proxies      = dihedral_proxies,
-    hd_selection          = hd_selection,
-    sites_cart            = sites_cart)
+    geometry_restraints = geometry_restraints,
+    bond_proxies        = bond_proxies_simple,
+    angle_proxies       = angle_proxies,
+    dihedral_proxies    = dihedral_proxies,
+    hd_selection        = hd_selection,
+    sites_cart          = sites_cart,
+    atoms               = atoms)
+
   h_parameterization = hydrogen_parametrization.get_h_parameterization(
     connectivity   = connectivity,
     sites_cart     = sites_cart,

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25568
          http://sourceforge.net/p/cctbx/code/25568
Author:   phyy-nx
Date:     2016-10-10 22:58:14 +0000 (Mon, 10 Oct 2016)
Log Message:
-----------
Sync with r25400, adding goniometer_shadow_mask to the rest of the format classes.

Revision Links:
--------------
    http://sourceforge.net/p/cctbx/code/25400

Modified Paths:
--------------
    trunk/dxtbx/format/FormatCBFMiniPilatusDLS12M.py
    trunk/dxtbx/format/FormatHDFEigerNearlyNexus.py
    trunk/dxtbx/format/FormatNexus.py
    trunk/dxtbx/format/FormatSMVADSC.py

Modified: trunk/dxtbx/format/FormatCBFMiniPilatusDLS12M.py
===================================================================
--- trunk/dxtbx/format/FormatCBFMiniPilatusDLS12M.py	2016-10-08 18:08:27 UTC (rev 25567)
+++ trunk/dxtbx/format/FormatCBFMiniPilatusDLS12M.py	2016-10-10 22:58:14 UTC (rev 25568)
@@ -313,7 +313,7 @@
     return GoniometerShadowMaskGenerator(
       goniometer, coords, flex.size_t(len(coords), 1))
 
-  def get_mask(self, goniometer=None):
+  def get_mask(self, goniometer_shadow_mask=False, goniometer=None):
     gonio_masker = self.get_goniometer_shadow_masker(goniometer=goniometer)
     scan = self.get_scan()
     detector = self.get_detector()

Modified: trunk/dxtbx/format/FormatHDFEigerNearlyNexus.py
===================================================================
--- trunk/dxtbx/format/FormatHDFEigerNearlyNexus.py	2016-10-08 18:08:27 UTC (rev 25567)
+++ trunk/dxtbx/format/FormatHDFEigerNearlyNexus.py	2016-10-10 22:58:14 UTC (rev 25568)
@@ -343,7 +343,7 @@
   def get_raw_data(self, index):
     return self._raw_data[index]
 
-  def get_mask(self, index=None):
+  def get_mask(self, index=None, goniometer_shadow_mask=False, goniometer=None):
     return self._mask
 
   def get_num_images(self):

Modified: trunk/dxtbx/format/FormatNexus.py
===================================================================
--- trunk/dxtbx/format/FormatNexus.py	2016-10-08 18:08:27 UTC (rev 25567)
+++ trunk/dxtbx/format/FormatNexus.py	2016-10-10 22:58:14 UTC (rev 25568)
@@ -109,7 +109,7 @@
   def get_raw_data(self, index):
     return self._raw_data[index]
 
-  def get_mask(self, index=None):
+  def get_mask(self, index=None, goniometer_shadow_mask=False, goniometer=None):
     return self._mask
 
   def get_num_images(self):

Modified: trunk/dxtbx/format/FormatSMVADSC.py
===================================================================
--- trunk/dxtbx/format/FormatSMVADSC.py	2016-10-08 18:08:27 UTC (rev 25567)
+++ trunk/dxtbx/format/FormatSMVADSC.py	2016-10-10 22:58:14 UTC (rev 25568)
@@ -154,7 +154,7 @@
 
     return raw_data
 
-  #def get_mask(self):
+  #def get_mask(self, goniometer_shadow_mask=False, goniometer=None):
     #from scitbx.array_family import flex
     #from iotbx.detectors import image_divider
 

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25567
          http://sourceforge.net/p/cctbx/code/25567
Author:   nksauter
Date:     2016-10-08 18:08:27 +0000 (Sat, 08 Oct 2016)
Log Message:
-----------
Documentation for the RS postrefinement model.

Added Paths:
-----------
    trunk/cctbx/examples/merging/samosa/Figure_1.pdf
    trunk/cctbx/examples/merging/samosa/Figure_2.pdf
    trunk/cctbx/examples/merging/samosa/postrefinement_rs_model.pdf
    trunk/cctbx/examples/merging/samosa/postrefinement_rs_model.tex

Added: trunk/cctbx/examples/merging/samosa/Figure_1.pdf
===================================================================
(Binary files differ)

Index: trunk/cctbx/examples/merging/samosa/Figure_1.pdf
===================================================================
--- trunk/cctbx/examples/merging/samosa/Figure_1.pdf	2016-10-07 21:55:31 UTC (rev 25566)
+++ trunk/cctbx/examples/merging/samosa/Figure_1.pdf	2016-10-08 18:08:27 UTC (rev 25567)

Property changes on: trunk/cctbx/examples/merging/samosa/Figure_1.pdf
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Added: trunk/cctbx/examples/merging/samosa/Figure_2.pdf
===================================================================
(Binary files differ)

Index: trunk/cctbx/examples/merging/samosa/Figure_2.pdf
===================================================================
--- trunk/cctbx/examples/merging/samosa/Figure_2.pdf	2016-10-07 21:55:31 UTC (rev 25566)
+++ trunk/cctbx/examples/merging/samosa/Figure_2.pdf	2016-10-08 18:08:27 UTC (rev 25567)

Property changes on: trunk/cctbx/examples/merging/samosa/Figure_2.pdf
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Added: trunk/cctbx/examples/merging/samosa/postrefinement_rs_model.pdf
===================================================================
(Binary files differ)

Index: trunk/cctbx/examples/merging/samosa/postrefinement_rs_model.pdf
===================================================================
--- trunk/cctbx/examples/merging/samosa/postrefinement_rs_model.pdf	2016-10-07 21:55:31 UTC (rev 25566)
+++ trunk/cctbx/examples/merging/samosa/postrefinement_rs_model.pdf	2016-10-08 18:08:27 UTC (rev 25567)

Property changes on: trunk/cctbx/examples/merging/samosa/postrefinement_rs_model.pdf
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Added: trunk/cctbx/examples/merging/samosa/postrefinement_rs_model.tex
===================================================================
--- trunk/cctbx/examples/merging/samosa/postrefinement_rs_model.tex	                        (rev 0)
+++ trunk/cctbx/examples/merging/samosa/postrefinement_rs_model.tex	2016-10-08 18:08:27 UTC (rev 25567)
@@ -0,0 +1,344 @@
+\documentclass[12pt, letterpaper]{article}
+\usepackage[english]{babel}
+\usepackage{amsmath}
+\usepackage{unicode-math}
+\usepackage[margin=0.7in]{geometry}
+\usepackage{graphicx}
+%\usepackage{mathrsfs}
+\usepackage{fontspec}
+\setmainfont{Calibri}
+% must be compiled with XeLaTeX, not pdfLaTeX
+
+  %%%%%%%%%%%%
+  % PREAMBLE %
+  %%%%%%%%%%%%
+\begin{document}
+\selectlanguage{english}
+
+\section*{Postrefinement: The model {\tt rs}. }
+  \par This is intended to be the simplest possible model for the reciprocal lattice point (RLP), 
+  describing the RLP as a sphere of radius ${r_s}$, which is globally constant over the whole dataset
+  consisting of an ensemble of crystal lattices (or frames).  
+\subsection*{1 The size of the RLP model}
+
+  \par The constant value of ${r_s}$ is computed as follows.
+  From model refinement and integration, each crystal has associated with it a list of Miller indices 
+  $\mathbf{h_i}$ and a reciprocal space orientation matrix $\mathbf{A}$ 
+  defined by Rossmann $\textit{et\ al.}$ (1979),
+    \begin{equation}
+    \mathbf{A} = 
+    \left(
+    \begin{array}{c c c}
+      a_{x}^{*}   &  b_{x}^{*}   & c_{x}^{*}  \\
+      a_{y}^{*}   &  b_{y}^{*}   & c_{y}^{*}  \\
+      a_{z}^{*}   &  b_{z}^{*}   & c_{z}^{*}  \\
+    \end{array}
+    \right)
+    \text{.}
+    \label{eqn:RossmannA}
+  \end{equation}
+  The reciprocal space coordinates of RLP $i$ are computed with 
+    \begin{equation}
+    \mathbf{q} = \mathbf{A}\mathbf{h}
+    \text{,}
+    \label{eqn:Ah}
+  \end{equation}
+  leading to a reciprocal position Q, with a small distance offset
+   ${r_h}$ away from the Ewald sphere that represents the perfect diffracting condition.  
+  The fact that $|{r_h}|$ is non-zreo is indicative that
+  Bragg observations from still shots represent partial reflections.  Note that array index $i$ denoting
+  a specific Miller index is dropped on occasion for clarity.  The geometry is explained in Fig. 1. 
+  
+  \begin{figure}[htb]
+  \begin{center}
+  \includegraphics[scale=1.5]{Figure_1.pdf}
+  \label{fig:1}
+  \end{center}
+  \begin{center}
+  {Fig. 1. Ewald sphere construction.}
+  \end{center}
+  \end{figure}
+
+  The quantity $|{r_h}|$ is given by 
+    \begin{equation}
+    {r_h} = ||\mathbf{q}+\mathbf{s_0}||-||\mathbf{s_1}|| = ||\mathbf{q}+\mathbf{s_0}||-\dfrac{1}{ \lambda}
+    \text{,}
+    \label{eqn:Rh}
+  \end{equation}
+
+  where ${s_0}$ and ${s_1}$ are respectively the beam vector and the diffracted ray vector,
+  each of length $1/\lambda$.  For the model {\tt rs}, the constant value of ${r_s}$ is taken 
+  as the root mean-squared value of
+  ${r_h}$ over all Bragg spots integrated from a given crystal.  It therefore depends on whatever
+  algorithm has been used to predict spots, regardless of whether there is measurable signal
+  in the spots.  
+  \subsection*{2 The geometry of the RLP model}
+
+  \par The intention is to create a model of the RLP similar to a hard sphere, so that if any portion 
+  of the sphere touches the Ewald sphere there is signal expected, otherwise none.  However, this 
+  is a discontinuous model (in terms of the spot partiality expressed as a function of ${r_h}$ and
+  therefore not easily amenable to parameter fitting.  Therefore we relax the requirement for a hard
+  sphere and adopt a radial profile somewhat smoother.  For the Uervirojnangkoorn (2015) paper
+  we used a profile based on a Lorentzian function.  The derivation is as follows.  
+  
+\par A suggestion from James Holton defines the Bragg spot partiality as
+  
+  \begin{equation}
+    p = \frac{\text{Area of intersection between the Ewald sphere and } F_{hkl}}
+             {\text{Area of intersection between the Ewald sphere and } F_{000}}
+    \text{.}
+    \label{eqn:part}
+  \end{equation}
+  
+The "areas of intersection" in question are really spherical caps that represent the Ewald 
+sphere's intersection with the reciprocal space ball of radius $r_s$.  However, we're not going to 
+insist on such detail; instead we will simply take a circular area of radius $r_p$ such that we
+have the right triangle
+
+  \begin{equation}
+    {r}_p^2 = {r}_s^2 - {r}_h^2
+    \text{,}
+    \label{eqn:pyth}
+  \end{equation}
+
+and then the approximate expression for partiality becomes (model A),
+
+  \begin{equation}
+    p_A = \frac{\pi r_p^2}{\pi r_s^2} = 1 - \frac{r_h^2}{r_s^2} \text{ for }|r_h|<r_s, 0 \text{ otherwise}
+    \text{.}
+    \label{eqn:fexprr}
+  \end{equation}
+
+Partiality as a function of $r_h$ is a simple inverted parabola with $p_A=1$ at $r_h=0$ and 
+roots at $\pm r_s$ (Fig. 2).  Outside of this domain the partiality is 0. 
+
+  \begin{figure}[htb!]
+  \begin{center}
+  \includegraphics[scale=0.65]{Figure_2.pdf}
+  %\includesvg{Figure_2}
+  \label{fig:2}
+  \end{center}
+  \begin{center}
+  {Fig. 2. Two partiality models:  a simple hard-sphere model ($p_A$, blue), and a soft-sphere Lorentzian
+  function ($p_B$, red).}
+  \end{center}
+  \end{figure}
+
+However, having a 
+mathematically discontinuous
+expression ($p_A$) will leave us at a disadvantage for postrefinement.  The postrefinement strategy  
+will be to express the lack of closure $r_h$ in terms of model parameters such as the unit cell
+dimensions and crystal orientation.  Then optimize a target function $f$ expressed in 
+terms of the partiality $p$, attempting to find parameter values to minimize $f$.  It is 
+crucial in this procedure to have an expression for partiality that is smooth and differentiable. 
+
+We will therefore have to modify our simple model of the Bragg spot as a reciprocal space ball.  
+One functional form that might have the desired properties is the Lorentzian function
+  \begin{equation}
+    L = \frac{1}{\pi}\frac{\frac{1}{2}\Gamma}{x^2 + (\frac{1}{2}\Gamma)^2}
+    \text{,}
+    \label{eqn:loren}
+  \end{equation}
+where $\Gamma$ is the full-width at half maximum (FWHM).  
+
+Let's tinker with this expression so it conforms to expectation...first multiply by 
+a scaling constant to get $L'(0)=1$:
+  \begin{equation}
+    L^{\prime} = \frac{\pi \Gamma}{2}L
+    \text{,}
+    \label{eqn:lorenA}
+  \end{equation}
+
+and finally setting the FWHM to the FWHM value obtained from eqn \eqref{eqn:fexprr},
+  \begin{equation}
+    \Gamma = \frac{2r_s}{\sqrt{2}}
+    \text{,}
+    \label{eqn:fwhm}
+  \end{equation}
+so we get a new partiality expression (model B):
+
+  \begin{equation}
+    p_B = \frac{r_s^2}{2r_h^2 + r_s^2}
+    \text{.}
+    \label{eqn:pb}
+  \end{equation}
+
+Finally, for postrefinement we'll need the partial derivative of $p_B$ with respect to $r_h$ (use the 
+quotient rule):
+  \begin{equation}
+    \frac{\partial{p_B}}{\partial{r_h}} = \frac{-4r_s^2r_h}{(2r_h^2 + r_s^2)^2}
+    \text{.}
+    \label{eqn:deriv}
+  \end{equation}
+
+
+  \subsection*{3 Model parameters and target function}
+  \par The goal of this work is to refine the parameters of the partiality model so that the 
+  observed intensities, corrected to their full spot equivalents, offer the best agreement over
+  repeated measurements of the same asymmetric-unit Miller index.  In practice,
+  the parameters representing each crystal lattice are refined 
+  against a set of reference intensities $I_{\mathrm{ref}}$.  
+  Program {\it prime} uses simple scaling to create an initial reference, after which repeated 
+  cycles of postrefinement are performed, with the reference being created from the corrected, 
+  merged intensities from the previous cycle.  In {\it cxi.merge} the reference is an isomorphous
+  atomic structure, from which intensities $I_{\mathrm{ref}}$ are calculated, and only one 
+  cycle is performed.  The polarization-corrected measurements are denoted $I_{\mathrm{obs}}$.
+  The parameters refined for each crystal lattice are:
+  
+  \par $G$: the scale factor.
+  \par $B$: the Wilson $B$-factor.
+  \par $\theta_{x}$: incremental crystal rotation angle on $x$-axis ($\perp$ to beam).
+  \par $\theta_{y}$: incremental crystal rotation angle on $y$-axis ($\perp$ to beam).
+  \par The least-squares target function used to achieve best agreement between model and 
+  observation is 
+  
+  
+  \begin{equation}
+ \mathscr{F} =  \sum_{i} \limits
+    ( G  \exp(\dfrac{-8B\sin^2\theta}{\lambda^2}) p_B I_{\mathrm{ref}} - I_{\mathrm{obs}})^{2}
+\end{equation}
+where $\theta$ is the Bragg diffraction angle defined in Fig. 1, and $\lambda$ the wavelength, both 
+treated as constants, and the sum is over all measurements integrated from a single crystal lattice. 
+
+  \subsection*{4 Necessary derivatives for parameter refinement}
+
+  \par Given the least-squares form, derivatives of the target functional with respect to 
+  parameter $\mathscr{P}$ are in general
+  
+    \begin{equation}
+    \frac{\partial\mathscr{F}}{\partial\mathscr{P}} = 2\sum_{i} \limits
+    \mathscr{R}_i\dfrac{\partial\mathscr{R}_i}{\partial\mathscr{P}}
+    \text{,}
+    \label{eqn:genFP}
+  \end{equation}
+where the residual comparison on each observation is 
+
+  \begin{equation}
+ \mathscr{R}_i =  
+    G  \exp(\dfrac{-8B\sin^2\theta}{\lambda^2}) p_B I_{\mathrm{ref}} - I_{\mathrm{obs}}
+    \text{.}
+    \end{equation}
+The derivatives in the Jacobian matrix, required for parameter optimization, are more-or-less
+straightforward:
+
+    \begin{equation}
+    \frac{\partial\mathscr{R}_i}{\partial G} = 
+    \exp(\dfrac{-8B\sin^2\theta}{\lambda^2}) p_B I_{\mathrm{ref}}
+    \text{,}
+    \label{eqn:dG}
+  \end{equation}
+
+     \begin{equation}
+    \frac{\partial\mathscr{R}_i}{\partial B} = 
+    G  \exp(\dfrac{-8B\sin^2\theta}{\lambda^2}) p_B I_{\mathrm{ref}}
+    \left( \dfrac{-8\sin^2\theta}{\lambda^2}\right)
+    \text{.}
+    \label{eqn:dB}
+  \end{equation}
+
+  \par The derivatives with respect to $\theta_{x}$ and $\theta_{y}$ require more work.  All of the
+  dependence on crystal orientation comes through the expression for partiality:
+  
+      \begin{equation}
+    \frac{\partial\mathscr{R}_i}{\partial \theta_{x\mathrm{|}y}} = 
+  \frac{\partial\mathscr{R}_i}{\partial p_B}
+   \frac{\partial p_B} {\partial \theta_{x\mathrm{|}y}}
+    \text{,}
+    \label{eqn:dthxy}
+  \end{equation}
+
+with
+
+      \begin{equation} 
+  \frac{\partial\mathscr{R}_i}{\partial p_B} =
+  G  \exp(\dfrac{-8B\sin^2\theta}{\lambda^2}) I_{\mathrm{ref}}
+    \text{.}
+    \label{eqn:P_terms}
+  \end{equation}
+
+As for the variation of the partiality model $p_B$ defined in \eqref{eqn:pb}, the {\tt rs}
+model assumes that the sphere radius $r_s$ is fixed, thus the only remaining variable is the
+distance  $r_h$ between RLP and Ewald sphere:
+
+   \begin{equation}
+   \frac{\partial p_B} {\partial \theta_{x\mathrm{|}y}} = 
+  \frac{\partial p_B}{\partial r_h}
+   \frac{\partial r_h} {\partial \theta_{x\mathrm{|}y}}
+    \text{.}
+    \label{eqn:dpb2}
+  \end{equation}
+
+
+  \par An expression for $\dfrac{\partial p_B}{\partial r_h}$ has already been 
+  given in \eqref{eqn:deriv}, so it now
+  remains to investigate the derivative $\dfrac{\partial r_h} {\partial \theta_{x\mathrm{|}y}}$,
+  based on the definition of $r_h$ given in \eqref{eqn:Rh}.
+
+Introduce the vector $\mathbf{S}$:
+   \begin{equation}
+   \mathbf{S} = \mathbf{q}+\mathbf{s_0}
+    \text{,}
+    \label{eqn:SS}
+  \end{equation}
+
+   \begin{equation}
+ {r_h} = ||\mathbf{S}||-\dfrac{1}{ \lambda}
+     \text{,}
+    \label{eqn:sss}
+  \end{equation}
+  
+    \begin{equation}
+   \dfrac{\partial r_h} {\partial \theta_{x\mathrm{|}y}} = 
+   \dfrac
+   {\mathbf{S}\cdot{
+   \dfrac{\partial \mathbf{S}} {\partial \theta_{x\mathrm{|}y}}
+   }}
+   {||\mathbf{S}||}
+    \text{,}
+    \label{eqn:rhth}
+  \end{equation}
+
+    \begin{equation}
+   \dfrac{\partial \mathbf{S}} {\partial \theta_{x\mathrm{|}y}} = 
+  {
+   \dfrac{\partial \mathbf{q}} {\partial \theta_{x\mathrm{|}y}}
+   }
+    \text{.}
+    \label{eqn:derivsq}
+  \end{equation}
+
+Finally we investigate the derivative of the RLP position $\mathbf{q}$ with respect to the crystal 
+rotations.  The effective orientation matrix $\mathbf{A}$ may be expressed as the 
+reference orientation matrix $\mathbf{A}_\mathrm{ref}$ determined during crystal refinement and 
+integration, composed with additional rotational operators $\mathbb{R}_x$ and 
+$\mathbb{R}_y$ determined by the postrefined angles $\theta_{x}$ and 
+$\theta_{y}$:
+
+    \begin{equation}
+   \mathbf{A} = 
+\mathbb{R}_y( \mathbb{R}_x ( \mathbf{A}_\mathrm{ref} )  )
+    \text{.}
+    \label{eqn:compoase}
+  \end{equation}
+
+The derivatives of $\mathbf{q}$ work out as follows:
+
+  \begin{equation}
+   \dfrac{\partial \mathbf{q}} {\partial \theta_{x}} = 
+  \mathbb{R}_y \dfrac{\partial\mathbb{R}_x}{\partial \theta_{x}}  \mathbf{A}_\mathrm{ref} \mathbf{h}
+    \text{,}
+    \label{eqn:qthx}
+  \end{equation}
+  \begin{equation}
+   \dfrac{\partial \mathbf{q}} {\partial \theta_{y}} = 
+  \dfrac{\partial\mathbb{R}_y}{\partial \theta_{y}} \mathbb{R}_x \mathbf{A}_\mathrm{ref} \mathbf{h}
+    \text{.}
+    \label{eqn:qthy}
+  \end{equation}
+
+The derivatives of the rotation operator are already encoded in the cctbx library ({\tt scitbx/matrix/\_\_init\_\_.py}).  Formulae for the rotation operator and its derivative with 
+respect to angle $\theta$ are given in the LaTeX documentation included in that directory.
+%%%%%%%%%%%%%%%%%%%%%
+
+
+\end{document}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25566
          http://sourceforge.net/p/cctbx/code/25566
Author:   olegsobolev
Date:     2016-10-07 21:55:31 +0000 (Fri, 07 Oct 2016)
Log Message:
-----------
One weight calculation for rsr. Update pair proxies and rama proxies in GRM before refinement. Surprisingly, this does not happen automatically in my case.

Modified Paths:
--------------
    trunk/mmtbx/refinement/geometry_minimization.py
    trunk/mmtbx/refinement/real_space/individual_sites.py
    trunk/mmtbx/secondary_structure/build/__init__.py

Modified: trunk/mmtbx/refinement/geometry_minimization.py
===================================================================
--- trunk/mmtbx/refinement/geometry_minimization.py	2016-10-07 21:44:59 UTC (rev 25565)
+++ trunk/mmtbx/refinement/geometry_minimization.py	2016-10-07 21:55:31 UTC (rev 25566)
@@ -460,6 +460,12 @@
 
     grm = get_geometry_restraints_manager(
         processed_pdb_file, xrs, params=params)
+  else:
+    grm.geometry.pair_proxies(
+        sites_cart=hierarchy.atoms().extract_xyz())
+    if grm.geometry.ramachandran_manager is not None:
+      grm.geometry.ramachandran_manager.update_phi_psi_targets(
+          sites_cart=hierarchy.atoms().extract_xyz())
 
   if reference_rotamers and original_pdb_h is not None:
     # make selection excluding rotamer outliers

Modified: trunk/mmtbx/refinement/real_space/individual_sites.py
===================================================================
--- trunk/mmtbx/refinement/real_space/individual_sites.py	2016-10-07 21:44:59 UTC (rev 25565)
+++ trunk/mmtbx/refinement/real_space/individual_sites.py	2016-10-07 21:55:31 UTC (rev 25566)
@@ -471,6 +471,13 @@
             ss_manager=ss_manager,
             hierarchy=self.pdb_h,
             log=self.log)
+    else:
+      self.grm.geometry.pair_proxies(
+          sites_cart=self.pdb_h.atoms().extract_xyz())
+      if self.grm.geometry.ramachandran_manager is not None:
+        self.grm.geometry.ramachandran_manager.update_phi_psi_targets(
+            sites_cart=self.pdb_h.atoms().extract_xyz())
+
     ncs_groups=None
     if len(ncs_restraints_group_list) > 0:
       ncs_groups=ncs_restraints_group_list
@@ -486,6 +493,7 @@
     selection_real_space = xrs.backbone_selection()
     # selection_real_space = None
     import mmtbx.refinement.real_space.weight
+    self.w = None
     for x in xrange(number_of_cycles):
       print >> self.log, "  Updating rotamer restraints..."
       self.pdb_h, grm = add_rotamer_restraints(
@@ -503,28 +511,29 @@
       # if True:
       if ncs_restraints_group_list is None or len(ncs_restraints_group_list)==0:
         #No NCS
-        print >> self.log, "  Determining weight...",
-        self.log.flush()
-        self.weight = mmtbx.refinement.real_space.weight.run(
-            map_data                    = target_map,
-            xray_structure              = self.xrs,
-            pdb_hierarchy               = self.pdb_h,
-            geometry_restraints_manager = grm,
-            rms_bonds_limit             = 0.015,
-            rms_angles_limit            = 1.0)
-        # division supposed to put more weight onto restraints. Need checking.
-        self.w = self.weight.weight/3.0
-        # self.w = self.weight.weight
-        # self.w =0.001
-        print >> self.log, self.w
-        # for s in self.weight.msg_strings:
-        #   print >> self.log, s
+        if self.w is None:
+          print >> self.log, "  Determining weight..."
+          self.log.flush()
+          self.weight = mmtbx.refinement.real_space.weight.run(
+              map_data                    = target_map,
+              xray_structure              = self.xrs,
+              pdb_hierarchy               = self.pdb_h,
+              geometry_restraints_manager = grm,
+              rms_bonds_limit             = 0.015,
+              rms_angles_limit            = 1.0)
+          # division is to put more weight onto restraints. Checked. Works.
+          self.w = self.weight.weight/3.0
+          # self.w = self.weight.weight
+          # self.w =2
+          # print >> self.log, self.w
+          for s in self.weight.msg_strings:
+            print >> self.log, s
         print >> self.log, "  Minimizing..."
         print >> self.log, "     with weight %f" % self.w
         self.log.flush()
         refine_object = simple(
             target_map                  = target_map,
-              selection                   = None,
+            selection                   = None,
             max_iterations              = 150,
             geometry_restraints_manager = grm.geometry,
             selection_real_space        = selection_real_space,
@@ -546,19 +555,20 @@
             xray_structure = self.xrs,
             map_data       = target_map,
             d_min          = 3)
-        print >> self.log, "  Determining weight... (NCS)",
-        self.weight = mmtbx.refinement.real_space.weight.run(
-            map_data                    = target_map,
-            xray_structure              = self.xrs,#.select(sel_master),
-            pdb_hierarchy               = self.pdb_h,#.select(sel_master),
-            geometry_restraints_manager = grm,
-            rms_bonds_limit             = 0.01,
-            rms_angles_limit            = 1.0,
-            ncs_groups                  = ncs_restraints_group_list)
-        # division supposed to put more weight onto restraints. Need checking.
-        self.w = self.weight.weight/3.0
-        for s in self.weight.msg_strings:
-          print >> self.log, s
+        if self.w is None:
+          print >> self.log, "  Determining weight... (NCS)",
+          self.weight = mmtbx.refinement.real_space.weight.run(
+              map_data                    = target_map,
+              xray_structure              = self.xrs,#.select(sel_master),
+              pdb_hierarchy               = self.pdb_h,#.select(sel_master),
+              geometry_restraints_manager = grm,
+              rms_bonds_limit             = 0.01,
+              rms_angles_limit            = 1.0,
+              ncs_groups                  = ncs_restraints_group_list)
+          # division supposed to put more weight onto restraints. Need checking.
+          self.w = self.weight.weight/3.0
+          for s in self.weight.msg_strings:
+            print >> self.log, s
         print >> self.log, "  Minimizing... (NCS)"
 
         actions = [[True, False], ]

Modified: trunk/mmtbx/secondary_structure/build/__init__.py
===================================================================
--- trunk/mmtbx/secondary_structure/build/__init__.py	2016-10-07 21:44:59 UTC (rev 25565)
+++ trunk/mmtbx/secondary_structure/build/__init__.py	2016-10-07 21:55:31 UTC (rev 25566)
@@ -695,6 +695,10 @@
   real_h.atoms().set_xyz(pre_result_h.atoms().extract_xyz())
   t9 = time()
   if processed_params.file_name_before_regularization is not None:
+    grm.geometry.pair_proxies(sites_cart=real_h.atoms().extract_xyz())
+    if grm.geometry.ramachandran_manager is not None:
+      grm.geometry.ramachandran_manager.update_phi_psi_targets(
+          sites_cart=real_h.atoms().extract_xyz())
     print >> log, "Outputting model before regularization %s" % processed_params.file_name_before_regularization
     real_h.write_pdb_file(
         file_name=processed_params.file_name_before_regularization)

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25565
          http://sourceforge.net/p/cctbx/code/25565
Author:   nksauter
Date:     2016-10-07 21:44:59 +0000 (Fri, 07 Oct 2016)
Log Message:
-----------
cxi.merge postrefinement: switch from finite-difference derivatives to analytical.

Modified Paths:
--------------
    trunk/cctbx/examples/merging/samosa/per_lattice_postrefinement.py

Modified: trunk/cctbx/examples/merging/samosa/per_lattice_postrefinement.py
===================================================================
--- trunk/cctbx/examples/merging/samosa/per_lattice_postrefinement.py	2016-10-07 18:05:01 UTC (rev 25564)
+++ trunk/cctbx/examples/merging/samosa/per_lattice_postrefinement.py	2016-10-07 21:44:59 UTC (rev 25565)
@@ -129,7 +129,7 @@
     self.observations_original_index_pair1_selected = observations_original_index_pair1_selected
 
   def run_plain(self):
-    self.MINI = lbfgs_minimizer_base( current_x = self.current,
+    self.MINI = lbfgs_minimizer_derivatives( current_x = self.current,
         parameterization = self.parameterization_class, refinery = self.refinery,
         out = self.out )
 
@@ -182,6 +182,15 @@
         Rh.append(Svec.length() - (1./self.WAVE))
       return Rh
 
+    def get_s1_array(self, values):
+      miller_vec = self.MILLER.as_vec3_double()
+      ref_ori = matrix.sqr(self.ORI.reciprocal_matrix())
+      Rx = matrix.col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(values.thetax)
+      Ry = matrix.col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(values.thetay)
+      s_array = flex.mat3_double(len(self.MILLER),Ry * Rx * ref_ori) * miller_vec
+      s1_array = s_array + flex.vec3_double(len(self.MILLER), self.BEAM)
+      return s1_array
+
     def get_eff_Astar(self, values):
       thetax = values.thetax; thetay = values.thetay;
       effective_orientation = self.ORI.rotate_thru((1,0,0),thetax
@@ -213,8 +222,86 @@
       Rh = self.get_Rh_array(values)
       rs_sq = rs*rs
       PB = rs_sq / ((2. * (Rh * Rh)) + rs_sq)
+      """
+      hard_sphere_partial =(rs_sq-(Rh*Rh))/rs_sq
+      immersion = Rh/rs
+      from matplotlib import pyplot as plt
+      plt.plot (immersion, PB, "r.")
+      plt.plot (immersion, hard_sphere_partial,"b.")
+      plt.plot ([-1.0,1.0],[0,0],"k-")
+      plt.show()
+      """
       return PB
 
+    def jacobian_callable(self,values):
+      PB = self.get_partiality_array(values)
+      EXP = flex.exp(-2.*values.BFACTOR*self.DSSQ)
+      G_terms = (EXP * PB * self.ICALCVEC)
+      B_terms = (values.G * EXP * PB * self.ICALCVEC)*(-2.*self.DSSQ)
+      P_terms = (values.G * EXP * self.ICALCVEC)
+
+      thetax = values.thetax; thetay = values.thetay;
+      Rx = matrix.col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(thetax)
+      dRx_dthetax = matrix.col((1,0,0)).axis_and_angle_as_r3_derivative_wrt_angle(thetax)
+      Ry = matrix.col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(thetay)
+      dRy_dthetay = matrix.col((0,1,0)).axis_and_angle_as_r3_derivative_wrt_angle(thetay)
+      ref_ori = matrix.sqr(self.ORI.reciprocal_matrix())
+      miller_vec = self.MILLER.as_vec3_double()
+      ds1_dthetax = flex.mat3_double(len(self.MILLER),Ry * dRx_dthetax * ref_ori) * miller_vec
+      ds1_dthetay = flex.mat3_double(len(self.MILLER),dRy_dthetay * Rx * ref_ori) * miller_vec
+
+      s1vec = self.get_s1_array(values)
+      s1lenvec = flex.sqrt(s1vec.dot(s1vec))
+      dRh_dthetax = s1vec.dot(ds1_dthetax)/s1lenvec
+      dRh_dthetay = s1vec.dot(ds1_dthetay)/s1lenvec
+      rs = values.RS
+      Rh = self.get_Rh_array(values)
+      rs_sq = rs*rs
+      dPB_dRh = -PB * 4. * Rh / (2. * Rh * Rh + rs_sq)
+      dPB_dthetax = dPB_dRh * dRh_dthetax
+      dPB_dthetay = dPB_dRh * dRh_dthetay
+      Px_terms = P_terms * dPB_dthetax; Py_terms = P_terms * dPB_dthetay
+
+      return [G_terms,B_terms,0,Px_terms,Py_terms]
+
+class nave1_refinery(rs_refinery):
+
+    def jacobian_callable(self,values):
+      PB = self.get_partiality_array(values)
+      EXP = flex.exp(-2.*values.BFACTOR*self.DSSQ)
+      G_terms = (EXP * PB * self.ICALCVEC)
+      B_terms = (values.G * EXP * PB * self.ICALCVEC)*(-2.*self.DSSQ)
+      P_terms = (values.G * EXP * self.ICALCVEC)
+
+      thetax = values.thetax; thetay = values.thetay;
+      Rx = matrix.col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(thetax)
+      dRx_dthetax = matrix.col((1,0,0)).axis_and_angle_as_r3_derivative_wrt_angle(thetax)
+      Ry = matrix.col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(thetay)
+      dRy_dthetay = matrix.col((0,1,0)).axis_and_angle_as_r3_derivative_wrt_angle(thetay)
+      ref_ori = matrix.sqr(self.ORI.reciprocal_matrix())
+      miller_vec = self.MILLER.as_vec3_double()
+      ds1_dthetax = flex.mat3_double(len(self.MILLER),Ry * dRx_dthetax * ref_ori) * miller_vec
+      ds1_dthetay = flex.mat3_double(len(self.MILLER),dRy_dthetay * Rx * ref_ori) * miller_vec
+
+      s1vec = self.get_s1_array(values)
+      s1lenvec = flex.sqrt(s1vec.dot(s1vec))
+      dRh_dthetax = s1vec.dot(ds1_dthetax)/s1lenvec
+      dRh_dthetay = s1vec.dot(ds1_dthetay)/s1lenvec
+      rs = values.RS
+      Rh = self.get_Rh_array(values)
+      rs_sq = rs*rs
+      denomin = (2. * Rh * Rh + rs_sq)
+      dPB_dRh = -PB * 4. * Rh / denomin
+      dPB_dthetax = dPB_dRh * dRh_dthetax
+      dPB_dthetay = dPB_dRh * dRh_dthetay
+      Px_terms = P_terms * dPB_dthetax; Py_terms = P_terms * dPB_dthetay
+
+      dPB_drs = 4 * rs * Rh * Rh / (denomin * denomin)
+      Prs_terms = P_terms * dPB_drs
+
+      return [G_terms,B_terms,Prs_terms,Px_terms,Py_terms]
+
+
 class eta_deff_refinery(refinery_base):
     def __init__(self, **kwargs):
       refinery_base.__init__(self,**kwargs)
@@ -332,3 +419,54 @@
     print >> self.out, "FINALMODEL",
     print >> self.out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)),
     values.show(self.out)
+
+class lbfgs_minimizer_derivatives(lbfgs_minimizer_base):
+
+  def compute_functional_and_gradients_test_code(self):
+    values = self.parameterization(self.x)
+    assert -150. < values.BFACTOR < 150. # limits on the exponent, please
+    self.func = self.refinery.fvec_callable(values)
+    functional = flex.sum(self.func*self.func)
+    self.f = functional
+    jacobian = self.refinery.jacobian_callable(values)
+    self.gg_0 = flex.sum(2. * self.func * jacobian[0])
+    self.gg_1 = flex.sum(2. * self.func * jacobian[1])
+    self.gg_3 = flex.sum(2. * self.func * jacobian[3])
+    self.gg_4 = flex.sum(2. * self.func * jacobian[4])
+    DELTA = 1.E-7
+    self.g = flex.double()
+    for x in xrange(self.n):
+      templist = list(self.x)
+      templist[x]+=DELTA
+      dvalues = flex.double(templist)
+
+      dfunc = self.refinery.fvec_callable(self.parameterization(dvalues))
+      dfunctional = flex.sum(dfunc*dfunc)
+      #calculate by finite_difference
+      self.g.append( ( dfunctional-functional )/DELTA )
+    self.g[2]=0.
+
+    print >> self.out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)),
+    values.show(self.out)
+    print >>self.out, "derivatives--> %15.5f    %15.5f    %9.7f   %5.2f   %5.2f"%tuple(self.g)
+    print >>self.out, "  analytical-> %15.5f    %15.5f                %5.2f   %5.2f"%(
+      self.gg_0,self.gg_1, self.gg_3,self.gg_4)
+    self.g[0]=self.gg_0
+    self.g[1]=self.gg_1
+    self.g[3]=self.gg_3
+    self.g[4]=self.gg_4
+    return self.f, self.g
+  def compute_functional_and_gradients(self):
+    values = self.parameterization(self.x)
+    assert -150. < values.BFACTOR < 150. # limits on the exponent, please
+    self.func = self.refinery.fvec_callable(values)
+    functional = flex.sum(self.func*self.func)
+    self.f = functional
+    jacobian = self.refinery.jacobian_callable(values)
+    self.g = flex.double(self.n)
+    for ix in xrange(self.n):
+      self.g[ix] = flex.sum(2. * self.func * jacobian[ix])
+    print >> self.out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)),
+    values.show(self.out)
+    return self.f, self.g
+

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25564
          http://sourceforge.net/p/cctbx/code/25564
Author:   bkpoon
Date:     2016-10-07 18:05:01 +0000 (Fri, 07 Oct 2016)
Log Message:
-----------
Ignore repeated warnings from matplotlib about building font cache

Modified Paths:
--------------
    trunk/wxtbx/plots/__init__.py

Modified: trunk/wxtbx/plots/__init__.py
===================================================================
--- trunk/wxtbx/plots/__init__.py	2016-10-07 17:41:11 UTC (rev 25563)
+++ trunk/wxtbx/plots/__init__.py	2016-10-07 18:05:01 UTC (rev 25564)
@@ -11,6 +11,7 @@
 if (sys.version_info[2] >= 6) :
   import warnings
   warnings.simplefilter('ignore', DeprecationWarning)
+  warnings.simplefilter('ignore', UserWarning) # for matplotlib 1.5.1
 
 class plot_container (wx.BoxSizer, wxtbx.MouseWheelTransparencyMixin) :
   def __init__ (self,

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25563
          http://sourceforge.net/p/cctbx/code/25563
Author:   bkpoon
Date:     2016-10-07 17:41:11 +0000 (Fri, 07 Oct 2016)
Log Message:
-----------
Xtriage: restored table output for NZ and L tests

Modified Paths:
--------------
    trunk/mmtbx/scaling/twin_analyses.py

Modified: trunk/mmtbx/scaling/twin_analyses.py
===================================================================
--- trunk/mmtbx/scaling/twin_analyses.py	2016-10-07 00:11:33 UTC (rev 25562)
+++ trunk/mmtbx/scaling/twin_analyses.py	2016-10-07 17:41:11 UTC (rev 25563)
@@ -805,10 +805,9 @@
         "mean_diff_c" : math.fabs(self.mean_diff_c), })
     if (out.gui_output) :
       out.show_plot(self.table)
+      out.show_table(self.table)
     else :
-      pass # XXX is there any point to printing out the table?
-      # FIXME this needs to be narrower - equal_widths=False
-      #out.show_table(self.table)
+      out.show_table(self.table)
 
 class l_test (scaling.xtriage_analysis):
   """
@@ -887,8 +886,9 @@
 """ % { "ml_alpha" : self.ml_alpha, })
     if (out.gui_output) :
       out.show_plot(self.table)
+      out.show_table(self.table)
     else :
-      pass #out.show_table(self.table)
+      out.show_table(self.table)
 
     out.show("""\
  Reference:

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25562
          http://sourceforge.net/p/cctbx/code/25562
Author:   olegsobolev
Date:     2016-10-07 00:11:33 +0000 (Fri, 07 Oct 2016)
Log Message:
-----------
Removing the last GRM construction. Some debugging/diagnostic output. Move rotamer work into the macrocycle.

Modified Paths:
--------------
    trunk/mmtbx/building/loop_idealization.py
    trunk/mmtbx/command_line/model_idealization.py
    trunk/mmtbx/refinement/geometry_minimization.py
    trunk/mmtbx/refinement/real_space/individual_sites.py
    trunk/mmtbx/regression/model_idealization/tst_nomap_01.py
    trunk/mmtbx/regression/model_idealization/tst_nomap_02.py
    trunk/mmtbx/regression/model_idealization/tst_nomap_03.py
    trunk/mmtbx/regression/model_idealization/tst_nomap_04.py
    trunk/mmtbx/regression/model_idealization/tst_withmap_01.py
    trunk/mmtbx/regression/model_idealization/tst_withmap_02.py
    trunk/mmtbx/regression/model_idealization/tst_withmap_03.py
    trunk/mmtbx/regression/model_idealization/tst_withmap_04.py
    trunk/mmtbx/secondary_structure/build/__init__.py

Modified: trunk/mmtbx/building/loop_idealization.py
===================================================================
--- trunk/mmtbx/building/loop_idealization.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/building/loop_idealization.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -11,6 +11,7 @@
 from mmtbx.secondary_structure.build import side_chain_placement, \
     set_xyz_smart
 from mmtbx.refinement.geometry_minimization import minimize_wrapper_for_ramachandran
+from cctbx import maptbx
 
 import boost.python
 ext = boost.python.import_ext("mmtbx_validation_ramachandran_ext")
@@ -59,27 +60,42 @@
                params=None,
                secondary_structure_annotation=None,
                reference_map=None,
+               crystal_symmetry=None,
+               grm=None,
+               rama_manager=None,
+               rotamer_manager=None,
                log=null_out(),
                verbose=False):
     if len(pdb_hierarchy.models()) > 1:
       raise Sorry("Multi-model files are not supported")
     self.original_pdb_h = pdb_hierarchy
     self.secondary_structure_annotation=secondary_structure_annotation
-    xrs = pdb_hierarchy.extract_xray_structure()
     asc = pdb_hierarchy.atom_selection_cache()
+    self.xrs = pdb_hierarchy.extract_xray_structure(crystal_symmetry=crystal_symmetry)
     self.reference_map = reference_map
     self.resulting_pdb_h = pdb_hierarchy.deep_copy()
     self.resulting_pdb_h.reset_atom_i_seqs()
     self.params = self.process_params(params)
     self.log = log
     self.verbose = verbose
-    self.r = rama_eval()
-    self.rotamer_manager = RotamerEval()
+    self.grm = grm
+    self.r = rama_manager
+    if self.r is None:
+      self.r = rama_eval()
+    self.rotamer_manager = rotamer_manager
+    if self.rotamer_manager is None:
+      self.rotamer_manager = RotamerEval()
     ram = ramalyze.ramalyze(pdb_hierarchy=pdb_hierarchy)
     self.p_initial_rama_outliers = ram.out_percent
     self.p_before_minimization_rama_outliers = None
     self.p_after_minimiaztion_rama_outliers = None
+    n_inputs = [reference_map, crystal_symmetry].count(None)
+    if not (n_inputs == 0 or n_inputs == 2):
+      print >> log, "Need to have both map and symmetry info. Not using map."
+      self.reference_map = None
 
+
+
     berkeley_count = utils.list_rama_outliers_h(self.resulting_pdb_h).count("\n")
     self.berkeley_p_before_minimization_rama_outliers = \
         berkeley_count/float(self.resulting_pdb_h.overall_counts().n_residues)*100
@@ -156,16 +172,18 @@
         if self.reference_map is None:
           minimize_wrapper_for_ramachandran(
               hierarchy=self.resulting_pdb_h,
-              xrs=xrs,
+              xrs=self.xrs,
               original_pdb_h=self.original_pdb_h,
               excl_string_selection=self.ref_exclusion_selection,
+              grm=self.grm,
               log=None,
               ss_annotation=self.secondary_structure_annotation)
         else:
           mwwm = minimize_wrapper_with_map(
               pdb_h=self.resulting_pdb_h,
-              xrs=xrs,
+              xrs=self.xrs,
               target_map=self.reference_map,
+              grm=self.grm,
               ss_annotation=self.secondary_structure_annotation,
               log=self.log)
         # self.resulting_pdb_h.write_pdb_file(file_name="%s_all_minized.pdb" % self.params.output_prefix)
@@ -311,11 +329,17 @@
         # states = ccd_obj.states
         # if self.params.save_states:
         #   states.write(file_name="%s%s_%d_%s_%d_%i_states.pdb" % (chain_id, out_res_num, ccd_radius, change_all, change_radius, i))
+        map_target = 0
+        if self.reference_map is not None:
+          map_target = maptbx.real_space_target_simple(
+              unit_cell   = self.xrs.crystal_symmetry().unit_cell(),
+              density_map = self.reference_map,
+              sites_cart  = h.atoms().extract_xyz())
 
         mc_rmsd = get_main_chain_rmsd_range(moving_h, h, all_atoms=True)
         if self.verbose:
-          print >> self.log, "Resulting anchor and backbone RMSDs, n_iter for model %d:" % i,
-          print >> self.log, resulting_rmsd, ",", mc_rmsd, ",", n_iter
+          print >> self.log, "Resulting anchor and backbone RMSDs, mapcc, n_iter for model %d:" % i,
+          print >> self.log, resulting_rmsd, ",", mc_rmsd, ",", map_target, ",", n_iter
           self.log.flush()
         #
         # setting new coordinates
@@ -350,14 +374,14 @@
         #
         # finalizing with geometry_minimization
         #
-        all_results.append((moved_with_side_chains_h.deep_copy(), mc_rmsd, resulting_rmsd, n_iter))
+        all_results.append((moved_with_side_chains_h.deep_copy(), mc_rmsd, resulting_rmsd, map_target, n_iter))
         if self.ccd_solution_is_ok(
             anchor_rmsd=resulting_rmsd,
             mc_rmsd=mc_rmsd,
             ccd_radius=ccd_radius,
             change_all_angles=change_all,
             change_radius=change_radius):
-          print "Choosen result (mc_rmsd, anchor_rmsd, n_iter):", mc_rmsd, resulting_rmsd, n_iter
+          print "Choosen result (mc_rmsd, anchor_rmsd, map_target, n_iter):", mc_rmsd, resulting_rmsd, map_target, n_iter
           self.log.flush()
           if minimize:
             print >> self.log, "minimizing..."
@@ -369,12 +393,14 @@
                   xrs=xrs,
                   original_pdb_h=original_pdb_h,
                   log=self.log,
+                  grm=self.grm,
                   ss_annotation=self.secondary_structure_annotation)
             else:
               mwwm = minimize_wrapper_with_map(
                   pdb_h=moved_with_side_chains_h,
                   xrs=xrs,
                   target_map=self.reference_map,
+                  grm=self.grm,
                   ss_annotation=self.secondary_structure_annotation,
                   log=self.log)
           # moved_with_side_chains_h.write_pdb_file(

Modified: trunk/mmtbx/command_line/model_idealization.py
===================================================================
--- trunk/mmtbx/command_line/model_idealization.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/command_line/model_idealization.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -24,6 +24,8 @@
 from mmtbx.refinement.real_space.individual_sites import minimize_wrapper_with_map
 from mmtbx.pdbtools import truncate_to_poly_gly
 from mmtbx.monomer_library.pdb_interpretation import grand_master_phil_str
+from mmtbx.rotamer.rotamer_eval import RotamerEval
+from mmtbx_validation_ramachandran_ext import rama_eval
 
 turned_on_ss = ssb.ss_idealization_master_phil_str
 turned_on_ss = turned_on_ss.replace("enabled = False", "enabled = True")
@@ -48,7 +50,7 @@
   .type = str
 use_map_for_reference = True
   .type = bool
-reference_map_resolution = 5.
+reference_map_resolution = 5
   .type = float
 data_for_map = None
   .type = path
@@ -103,6 +105,8 @@
 
     self.mon_lib_srv = None
     self.ener_lib = None
+    self.rotamer_manager = None
+    self.rama_manager = rama_eval()
 
     self.original_hierarchy = None # original pdb_h, without any processing
     self.original_boxed_hierarchy = None # original and boxed (if needed)
@@ -201,6 +205,60 @@
     self.reference_map = fft_map.real_map_unpadded(in_place=False)
     fft_map.as_xplor_map(file_name="%s.map" % self.params.output_prefix)
 
+  def prepare_reference_map_2(self, xrs, pdb_h):
+    print >> self.log, "Preparing reference map, method 2"
+    # new_h = pdb_h.deep_copy()
+    # truncate_to_poly_gly(new_h)
+    # xrs = new_h.extract_xray_structure(crystal_symmetry=xrs.crystal_symmetry())
+    xrs=xrs.set_b_iso(value=50)
+
+    # side_chain_no_cb_selection = ~ xrs.main_chain_selection()
+    side_chain_no_cb_selection = ~ xrs.backbone_selection()
+    xrs = xrs.set_b_iso(value=200, selection=side_chain_no_cb_selection)
+
+    crystal_gridding = maptbx.crystal_gridding(
+        unit_cell        = xrs.unit_cell(),
+        space_group_info = xrs.space_group_info(),
+        symmetry_flags   = maptbx.use_space_group_symmetry,
+        d_min             = self.params.reference_map_resolution)
+    fc = xrs.structure_factors(d_min = self.params.reference_map_resolution, algorithm = "direct").f_calc()
+    fft_map = miller.fft_map(
+        crystal_gridding=crystal_gridding,
+        fourier_coefficients=fc)
+    fft_map.apply_sigma_scaling()
+    self.reference_map = fft_map.real_map_unpadded(in_place=False)
+    fft_map.as_xplor_map(file_name="%s_2.map" % self.params.output_prefix)
+
+  def prepare_reference_map_3(self, xrs, pdb_h):
+    """ with ramachandran outliers """
+
+    print >> self.log, "Preparing reference map, method 3"
+    outlier_selection_txt = mmtbx.building.loop_closure.utils. \
+          rama_outliers_selection(pdb_h, self.rama_manager, 1)
+    asc = pdb_h.atom_selection_cache()
+    print >> self.log, "rama outlier selection:", outlier_selection_txt
+    rama_out_sel = asc.selection(outlier_selection_txt)
+    xrs=xrs.set_b_iso(value=50)
+
+    # side_chain_no_cb_selection = ~ xrs.main_chain_selection()
+    side_chain_no_cb_selection = ~ xrs.backbone_selection()
+    xrs = xrs.set_b_iso(value=200, selection=side_chain_no_cb_selection)
+    xrs = xrs.set_b_iso(value=150, selection=rama_out_sel)
+    # xrs = xrs.set_occupancies(value=0.3, selection=rama_out_sel)
+
+    crystal_gridding = maptbx.crystal_gridding(
+        unit_cell        = xrs.unit_cell(),
+        space_group_info = xrs.space_group_info(),
+        symmetry_flags   = maptbx.use_space_group_symmetry,
+        d_min             = self.params.reference_map_resolution)
+    fc = xrs.structure_factors(d_min = self.params.reference_map_resolution, algorithm = "direct").f_calc()
+    fft_map = miller.fft_map(
+        crystal_gridding=crystal_gridding,
+        fourier_coefficients=fc)
+    fft_map.apply_sigma_scaling()
+    self.reference_map = fft_map.real_map_unpadded(in_place=False)
+    fft_map.as_xplor_map(file_name="%s_3.map" % self.params.output_prefix)
+
   def get_grm(self):
     # first make whole grm using self.whole_pdb_h
     params_line = grand_master_phil_str
@@ -228,6 +286,7 @@
 
     self.mon_lib_srv = processed_pdb_files_srv.mon_lib_srv
     self.ener_lib = processed_pdb_files_srv.ener_lib
+    self.rotamer_manager = RotamerEval(mon_lib_srv=self.mon_lib_srv)
 
     self.whole_grm = get_geometry_restraints_manager(
         processed_pdb_file, self.whole_xrs, params=params)
@@ -284,7 +343,10 @@
       self.whole_xrs = self.working_xrs
 
     if self.params.use_map_for_reference:
-      self.prepare_reference_map(xrs=self.whole_xrs, pdb_h=self.whole_pdb_h)
+      # self.prepare_reference_map(xrs=self.whole_xrs, pdb_h=self.whole_pdb_h)
+      # self.prepare_reference_map_2(xrs=self.whole_xrs, pdb_h=self.whole_pdb_h)
+      self.prepare_reference_map_3(xrs=self.whole_xrs, pdb_h=self.whole_pdb_h)
+    # STOP()
 
     # getting grm without SS restraints
     self.get_grm()
@@ -319,7 +381,7 @@
       negate_selection = None
       if self.reference_map is None:
         outlier_selection_txt = mmtbx.building.loop_closure.utils. \
-          rama_outliers_selection(self.working_pdb_h, None, 1)
+          rama_outliers_selection(self.working_pdb_h, self.rama_manager, 1)
         print >> self.log, "outlier_selection_txt", outlier_selection_txt
         negate_selection = "all"
         if outlier_selection_txt != "" and outlier_selection_txt is not None:
@@ -346,6 +408,7 @@
           cif_objects=self.cif_objects,
           verbose=True,
           reference_map=self.reference_map,
+          rotamer_manager=self.rotamer_manager,
           log=self.log)
       self.log.flush()
 
@@ -358,7 +421,8 @@
     # Write resulting pdb file.
     self.shift_and_write_result(
         hierarchy=self.working_pdb_h,
-        fname_suffix="ss_ideal")
+        fname_suffix="ss_ideal",
+        grm=self.working_grm)
     # STOP()
     self.params.loop_idealization.minimize_whole = not self.using_ncs
     # self.params.loop_idealization.enabled = False
@@ -368,13 +432,18 @@
         params=self.params.loop_idealization,
         secondary_structure_annotation=self.ann,
         reference_map=self.reference_map,
+        crystal_symmetry=self.working_xrs.crystal_symmetry(),
+        grm=self.working_grm,
+        rama_manager=self.rama_manager,
+        rotamer_manager=self.rotamer_manager,
         log=self.log,
-        verbose=False)
+        verbose=True)
     self.log.flush()
     # STOP()
     self.shift_and_write_result(
         hierarchy=loop_ideal.resulting_pdb_h,
-        fname_suffix="rama_ideal")
+        fname_suffix="rama_ideal",
+        grm=self.working_grm)
     self.after_loop_idealization = geometry_no_grm(
         pdb_hierarchy=iotbx.pdb.input(
           source_info=None,
@@ -388,22 +457,6 @@
         self.after_loop_idealization.rotamer_outliers > 0.004):
       print >> self.log, "Processing pdb file again for fixing rotamers..."
       self.log.flush()
-      # again get grm... - need to optimize somehow
-      processed_pdb_files_srv = mmtbx.utils.\
-          process_pdb_file_srv(
-              crystal_symmetry= self.cs,
-              pdb_interpretation_params = None,
-              stop_for_unknowns         = False,
-              log=self.log,
-              cif_objects=None)
-      processed_pdb_file, pdb_inp = processed_pdb_files_srv.\
-          process_pdb_files(raw_records=flex.split_lines(fixed_rot_pdb_h.as_pdb_string()))
-
-      grm = get_geometry_restraints_manager(
-          processed_pdb_file, self.working_xrs, params=None)
-
-      # mon_lib_srv and rotamer_manager already was created multiple times
-      # to this moment in different procedures :(
       print >> self.log, "Fixing rotamers..."
       self.log.flush()
       self.shift_and_write_result(
@@ -411,16 +464,17 @@
         fname_suffix="just_before_rota")
       fixed_rot_pdb_h = fix_rotamer_outliers(
           pdb_hierarchy=fixed_rot_pdb_h,
-          grm=grm.geometry,
+          grm=self.working_grm.geometry,
           xrs=self.working_xrs,
           map_data=self.reference_map,
           mon_lib_srv=self.mon_lib_srv,
-          rotamer_manager=None,
+          rotamer_manager=self.rotamer_manager,
           verbose=True)
 
     self.shift_and_write_result(
         hierarchy=fixed_rot_pdb_h,
-        fname_suffix="rota_ideal")
+        fname_suffix="rota_ideal",
+        grm=self.working_grm)
     cs_to_write = self.cs if self.shift_vector is None else None
     self.after_rotamer_fixing = geometry_no_grm(
         pdb_hierarchy=iotbx.pdb.input(
@@ -461,7 +515,8 @@
         reference_map = self.reference_map)
     self.shift_and_write_result(
         hierarchy=self.whole_pdb_h,
-        fname_suffix="all_idealized")
+        fname_suffix="all_idealized",
+        grm=self.whole_grm)
     self.final_model_statistics = geometry_no_grm(
         pdb_hierarchy=iotbx.pdb.input(
           source_info=None,
@@ -491,7 +546,8 @@
           ncs_restraints_group_list=ncs_restraints_group_list,
           ss_annotation=ss_annotation,
           mon_lib_srv=self.mon_lib_srv,
-          ener_lib=self.ener_lib)
+          ener_lib=self.ener_lib,
+          rotamer_manager=self.rotamer_manager)
     else:
       print >> self.log, "Using map as reference"
       self.log.flush()
@@ -501,12 +557,13 @@
           target_map=reference_map,
           grm=grm,
           mon_lib_srv=self.mon_lib_srv,
+          rotamer_manager=self.rotamer_manager,
           ncs_restraints_group_list=ncs_restraints_group_list,
           ss_annotation=ss_annotation,
           number_of_cycles=self.params.number_of_refinement_cycles,
           log=self.log)
 
-  def shift_and_write_result(self, hierarchy, fname_suffix):
+  def shift_and_write_result(self, hierarchy, fname_suffix, grm=None):
     cs_to_write = self.cs if self.shift_vector is None else None
     pdb_h_shifted = hierarchy.deep_copy()
     pdb_h_shifted.reset_atom_i_seqs()
@@ -524,6 +581,11 @@
         pdb_hierarchy=pdb_h_shifted,
         crystal_symmetry=cs_to_write,
         ss_annotation=self.original_ann)
+    # if grm is not None:
+    #   grm.write_geo_file(
+    #       sites_cart=hierarchy.atoms().extract_xyz(),
+    #       site_labels= [atom.id_str() for atom in hierarchy.atoms()],
+    #       file_name="%s_%s.geo" % (self.params.output_prefix, fname_suffix))
 
   def get_rmsd_from_start(self):
     if self.rmsd_from_start is not None:
@@ -550,6 +612,7 @@
         ("Clashscore", "clashscore", "{:10.2f}"),
         ("CBeta deviations", "c_beta_dev", "{:10d}"),
         ("Ramachandran outliers", "ramachandran_outliers", "{:10.2f}"),
+        ("Ramachandran allowed", "ramachandran_allowed", "{:10.2f}"),
         ("Rotamer outliers", "rotamer_outliers", "{:10.2f}"),
         ("Cis-prolines", "n_cis_proline", "{:10d}"),
         ("Cis-general", "n_cis_general", "{:10d}"),

Modified: trunk/mmtbx/refinement/geometry_minimization.py
===================================================================
--- trunk/mmtbx/refinement/geometry_minimization.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/refinement/geometry_minimization.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -398,6 +398,7 @@
     ss_annotation = None,
     mon_lib_srv=None,
     ener_lib=None,
+    rotamer_manager=None,
     reference_rotamers = True,
     number_of_cycles=1,
     run_first_minimization_without_reference=False,
@@ -464,7 +465,8 @@
     # make selection excluding rotamer outliers
     from mmtbx.rotamer.rotamer_eval import RotamerEval
     # print "Excluding rotamer outliers"
-    rotamer_manager = RotamerEval(mon_lib_srv=mon_lib_srv)
+    if rotamer_manager is None:
+      rotamer_manager = RotamerEval(mon_lib_srv=mon_lib_srv)
     non_rot_outliers_selection = flex.bool(hierarchy.atoms_size(), False)
     for model in original_pdb_h.models():
       for chain in model.chains():

Modified: trunk/mmtbx/refinement/real_space/individual_sites.py
===================================================================
--- trunk/mmtbx/refinement/real_space/individual_sites.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/refinement/real_space/individual_sites.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -397,13 +397,15 @@
       xrs,
       target_map,
       grm=None,
-      ncs_restraints_group_list=None,
+      ncs_restraints_group_list=[],
       mon_lib_srv=None,
+      rotamer_manager=None,
       ss_annotation=None,
       refine_ncs_operators=False,
       number_of_cycles=1,
       log=None):
     from mmtbx.refinement.geometry_minimization import add_rotamer_restraints
+    from mmtbx.model_statistics import geometry_no_grm
     self.pdb_h = pdb_h
     self.xrs = xrs
     self.log = log
@@ -455,46 +457,49 @@
       grm = get_geometry_restraints_manager(
           processed_pdb_file, xrs, params=params)
       # dealing with SS
-    if ss_annotation is not None:
-      from mmtbx.secondary_structure import manager
-      ss_manager = manager(
-          pdb_hierarchy=self.pdb_h,
-          geometry_restraints_manager=grm.geometry,
-          sec_str_from_pdb_file=ss_annotation,
-          params=None,
-          mon_lib_srv=mon_lib_srv,
-          verbose=-1,
-          log=self.log)
-      grm.geometry.set_secondary_structure_restraints(
-          ss_manager=ss_manager,
-          hierarchy=self.pdb_h,
-          log=self.log)
+      if ss_annotation is not None:
+        from mmtbx.secondary_structure import manager
+        ss_manager = manager(
+            pdb_hierarchy=self.pdb_h,
+            geometry_restraints_manager=grm.geometry,
+            sec_str_from_pdb_file=ss_annotation,
+            params=None,
+            mon_lib_srv=mon_lib_srv,
+            verbose=-1,
+            log=self.log)
+        grm.geometry.set_secondary_structure_restraints(
+            ss_manager=ss_manager,
+            hierarchy=self.pdb_h,
+            log=self.log)
     ncs_groups=None
     if len(ncs_restraints_group_list) > 0:
       ncs_groups=ncs_restraints_group_list
 
-    from mmtbx.rotamer.rotamer_eval import RotamerEval
-    rotamer_manager = RotamerEval(mon_lib_srv=mon_lib_srv)
+    if rotamer_manager is None:
+      from mmtbx.rotamer.rotamer_eval import RotamerEval
+      rotamer_manager = RotamerEval(mon_lib_srv=mon_lib_srv)
 
     self.pdb_h.write_pdb_file(file_name="rsr_before_rot_fix.pdb",
         crystal_symmetry=self.xrs.crystal_symmetry())
-    print >> self.log, "Making rotamer restraints..."
-    self.pdb_h, grm = add_rotamer_restraints(
-      pdb_hierarchy      = self.pdb_h,
-      restraints_manager = grm,
-      selection          = None,
-      sigma              = 10,
-      mode               = "fix_outliers",
-      accept_allowed     = False,
-      mon_lib_srv        = mon_lib_srv,
-      rotamer_manager    = rotamer_manager)
-    self.xrs = self.pdb_h.extract_xray_structure(crystal_symmetry=self.cs)
-    self.pdb_h.write_pdb_file(file_name="rsr_after_rot_fix.pdb",
-        crystal_symmetry=self.xrs.crystal_symmetry())
     # STOP()
 
+    selection_real_space = xrs.backbone_selection()
+    # selection_real_space = None
     import mmtbx.refinement.real_space.weight
     for x in xrange(number_of_cycles):
+      print >> self.log, "  Updating rotamer restraints..."
+      self.pdb_h, grm = add_rotamer_restraints(
+        pdb_hierarchy      = self.pdb_h,
+        restraints_manager = grm,
+        selection          = None,
+        sigma              = 5,
+        mode               = "fix_outliers",
+        accept_allowed     = False,
+        mon_lib_srv        = mon_lib_srv,
+        rotamer_manager    = rotamer_manager)
+      self.xrs = self.pdb_h.extract_xray_structure(crystal_symmetry=self.cs)
+      self.pdb_h.write_pdb_file(file_name="rsr_after_rot_fix.pdb",
+          crystal_symmetry=self.xrs.crystal_symmetry())
       # if True:
       if ncs_restraints_group_list is None or len(ncs_restraints_group_list)==0:
         #No NCS
@@ -505,21 +510,24 @@
             xray_structure              = self.xrs,
             pdb_hierarchy               = self.pdb_h,
             geometry_restraints_manager = grm,
-            rms_bonds_limit             = 0.01,
+            rms_bonds_limit             = 0.015,
             rms_angles_limit            = 1.0)
         # division supposed to put more weight onto restraints. Need checking.
         self.w = self.weight.weight/3.0
-        # self.w = 1
+        # self.w = self.weight.weight
+        # self.w =0.001
         print >> self.log, self.w
-        for s in self.weight.msg_strings:
-          print >> self.log, s
+        # for s in self.weight.msg_strings:
+        #   print >> self.log, s
         print >> self.log, "  Minimizing..."
+        print >> self.log, "     with weight %f" % self.w
         self.log.flush()
         refine_object = simple(
             target_map                  = target_map,
-            selection                   = None,
+              selection                   = None,
             max_iterations              = 150,
             geometry_restraints_manager = grm.geometry,
+            selection_real_space        = selection_real_space,
             states_accumulator          = None,
             ncs_groups                  = ncs_groups)
         refine_object.refine(weight = self.w, xray_structure = self.xrs)
@@ -582,7 +590,11 @@
           # self.structure_monitor.update(
           #   xray_structure = tfg_obj.xray_structure,
           #   accept_as_is   = True)
-    self.pdb_h.adopt_xray_structure(self.xrs)
+      self.pdb_h.adopt_xray_structure(self.xrs)
+      # ms = geometry_no_grm(
+      #     pdb_hierarchy=self.pdb_h,
+      #     molprobity_scores=True)
+      # print >> self.log, ms.format_molprobity_scores(prefix="    ")
 
 
     # print >> log, "pdb_h", self.pdb_h.atoms_size()

Modified: trunk/mmtbx/regression/model_idealization/tst_nomap_01.py
===================================================================
--- trunk/mmtbx/regression/model_idealization/tst_nomap_01.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/regression/model_idealization/tst_nomap_01.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -19,6 +19,7 @@
       "%s_start.pdb" % prefix,
       "use_map_for_reference=False",
       "number_of_refinement_cycles=1",
+      "n_macro=1",
       ">%s.log" % prefix])
   print cmd
   assert not easy_run.call(cmd)

Modified: trunk/mmtbx/regression/model_idealization/tst_nomap_02.py
===================================================================
--- trunk/mmtbx/regression/model_idealization/tst_nomap_02.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/regression/model_idealization/tst_nomap_02.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -22,6 +22,7 @@
       "%s_start.pdb" % prefix,
       "use_map_for_reference=False",
       "number_of_refinement_cycles=1",
+      "n_macro=1",
       ">%s.log" % prefix])
   print cmd
   assert not easy_run.call(cmd)

Modified: trunk/mmtbx/regression/model_idealization/tst_nomap_03.py
===================================================================
--- trunk/mmtbx/regression/model_idealization/tst_nomap_03.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/regression/model_idealization/tst_nomap_03.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -238,6 +238,7 @@
       "%s_start.pdb" % prefix,
       "number_of_refinement_cycles=1",
       "use_map_for_reference=False",
+      "n_macro=1",
       ">%s.log" % prefix])
   print cmd
   assert not easy_run.call(cmd)

Modified: trunk/mmtbx/regression/model_idealization/tst_nomap_04.py
===================================================================
--- trunk/mmtbx/regression/model_idealization/tst_nomap_04.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/regression/model_idealization/tst_nomap_04.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -231,6 +231,7 @@
       "%s_start.pdb" % prefix,
       "use_map_for_reference=False",
       "number_of_refinement_cycles=1",
+      "n_macro=1",
       ">%s.log" % prefix])
   print cmd
   assert not easy_run.call(cmd)

Modified: trunk/mmtbx/regression/model_idealization/tst_withmap_01.py
===================================================================
--- trunk/mmtbx/regression/model_idealization/tst_withmap_01.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/regression/model_idealization/tst_withmap_01.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -18,6 +18,7 @@
       "use_map_for_reference=True",
       "number_of_refinement_cycles=1",
       "loop_idealization.number_of_ccd_trials=1",
+      "n_macro=1",
       ">%s.log" % prefix])
   print cmd
   assert not easy_run.call(cmd)

Modified: trunk/mmtbx/regression/model_idealization/tst_withmap_02.py
===================================================================
--- trunk/mmtbx/regression/model_idealization/tst_withmap_02.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/regression/model_idealization/tst_withmap_02.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -22,6 +22,7 @@
       "use_map_for_reference=True",
       "loop_idealization.number_of_ccd_trials=1",
       "number_of_refinement_cycles=1",
+      "n_macro=1",
       "%s_start.pdb" % prefix,
       ">%s.log" % prefix])
   print cmd

Modified: trunk/mmtbx/regression/model_idealization/tst_withmap_03.py
===================================================================
--- trunk/mmtbx/regression/model_idealization/tst_withmap_03.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/regression/model_idealization/tst_withmap_03.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -242,6 +242,7 @@
       "use_map_for_reference=True",
       "loop_idealization.number_of_ccd_trials=1",
       "number_of_refinement_cycles=1",
+      "n_macro=1",
       ">%s.log" % prefix])
   print cmd
   assert not easy_run.call(cmd)

Modified: trunk/mmtbx/regression/model_idealization/tst_withmap_04.py
===================================================================
--- trunk/mmtbx/regression/model_idealization/tst_withmap_04.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/regression/model_idealization/tst_withmap_04.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -235,6 +235,7 @@
       "use_map_for_reference=True",
       "loop_idealization.number_of_ccd_trials=1",
       "number_of_refinement_cycles=1",
+      "n_macro=1",
       ">%s.log" % prefix])
   print cmd
   assert not easy_run.call(cmd)

Modified: trunk/mmtbx/secondary_structure/build/__init__.py
===================================================================
--- trunk/mmtbx/secondary_structure/build/__init__.py	2016-10-06 18:56:41 UTC (rev 25561)
+++ trunk/mmtbx/secondary_structure/build/__init__.py	2016-10-07 00:11:33 UTC (rev 25562)
@@ -12,6 +12,7 @@
 from mmtbx.command_line.geometry_minimization import \
   get_geometry_restraints_manager
 from time import time
+from mmtbx.refinement.real_space.individual_sites import minimize_wrapper_with_map
 
 alpha_helix_str = """
 ATOM      1  N   GLY A   1      -5.606  -2.251 -12.878  1.00  0.00           N
@@ -731,33 +732,19 @@
         fix_rotamer_outliers     = fix_rotamer_outliers,
         log                      = refinement_log)
   else:
-    from mmtbx.refinement.real_space.individual_sites import simple
-    import mmtbx.refinement.real_space.weight
-    print >> log, "  Determining weight..."
-    log.flush()
     ref_xrs = real_h.extract_xray_structure(
         crystal_symmetry=xray_structure.crystal_symmetry())
-    weight = mmtbx.refinement.real_space.weight.run(
-      map_data                    = reference_map,
-      xray_structure              = ref_xrs,
-      pdb_hierarchy               = real_h,
-      geometry_restraints_manager = grm,
-      rms_bonds_limit             = 0.02,
-      rms_angles_limit            = 2.0)
-    w = weight.weight
-    print >> log, "  Minimizing..."
-    log.flush()
-    refine_object = simple(
-      target_map                  = reference_map,
-      selection                   = None,
-      max_iterations              = 150,
-      geometry_restraints_manager = grm.geometry,
-      states_accumulator          = None,
-      ncs_groups                  = None)
-    refine_object.refine(weight = w, xray_structure = ref_xrs)
-    ref_xrs=ref_xrs.replace_sites_cart(
-      new_sites=refine_object.sites_cart(), selection=None)
-    real_h.adopt_xray_structure(ref_xrs)
+    minimize_wrapper_with_map(
+        pdb_h=real_h,
+        xrs=ref_xrs,
+        target_map=reference_map,
+        grm=grm,
+        ncs_restraints_group_list=[],
+        mon_lib_srv=None,
+        ss_annotation=ss_annotation,
+        refine_ncs_operators=False,
+        number_of_cycles=processed_params.n_macro,
+        log=log)
     real_h.write_pdb_file("after_ss_map_min.pdb")
 
   log.write(" Done\n")

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25561
          http://sourceforge.net/p/cctbx/code/25561
Author:   dcliebschner
Date:     2016-10-06 18:56:41 +0000 (Thu, 06 Oct 2016)
Log Message:
-----------
better handling for unknown H geometry

Modified Paths:
--------------
    trunk/mmtbx/hydrogens/hydrogen_parametrization.py

Modified: trunk/mmtbx/hydrogens/hydrogen_parametrization.py
===================================================================
--- trunk/mmtbx/hydrogens/hydrogen_parametrization.py	2016-10-06 17:28:13 UTC (rev 25560)
+++ trunk/mmtbx/hydrogens/hydrogen_parametrization.py	2016-10-06 18:56:41 UTC (rev 25561)
@@ -176,9 +176,14 @@
 
 
 # for every H atom, determine the type of bond
+# for debugging
+#def get_h_parameterization(connectivity, sites_cart, idealize, atoms_list, names):
 def get_h_parameterization(connectivity, sites_cart, idealize):
   h_parameterization = {}
   for ih in connectivity.keys():
+  # for debugging
+  #  print 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
+  #    atoms_list[ih].resseq, 'chain', atoms_list[ih].chain_id
     # if entry exists already, skip it
     if ih in h_parameterization:
       continue
@@ -371,6 +376,7 @@
           alpha  = alpha,
           dist_h = dist_h)
     else:
+      a1 = reduced_neighbs[0]
       h_parameterization[ih] = parameterization_info(
         htype  = 'unk',
         a0     = a0.iseq,
@@ -523,6 +529,13 @@
     connectivity   = connectivity,
     sites_cart     = sites_cart,
     idealize       = idealize)
+  # for debugging
+  #h_parameterization = get_h_parameterization(
+  #  connectivity   = connectivity,
+  #  sites_cart     = sites_cart,
+  #  idealize       = idealize,
+  #  atoms_list     = atoms_list,
+  #  names          = names)
 
   print >>log, '\nNow reconstructing H atoms...'
   long_distance_list = []

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25560
          http://sourceforge.net/p/cctbx/code/25560
Author:   dcliebschner
Date:     2016-10-06 17:28:13 +0000 (Thu, 06 Oct 2016)
Log Message:
-----------
create output file with new H

Modified Paths:
--------------
    trunk/mmtbx/hydrogens/hydrogen_parametrization.py

Modified: trunk/mmtbx/hydrogens/hydrogen_parametrization.py
===================================================================
--- trunk/mmtbx/hydrogens/hydrogen_parametrization.py	2016-10-05 23:30:29 UTC (rev 25559)
+++ trunk/mmtbx/hydrogens/hydrogen_parametrization.py	2016-10-06 17:28:13 UTC (rev 25560)
@@ -1,5 +1,5 @@
 from __future__ import division
-import sys
+import sys, os
 import time
 import mmtbx.monomer_library.server
 import mmtbx.monomer_library.pdb_interpretation
@@ -565,17 +565,18 @@
         sites_cart        = sites_cart,
         ih                = ih,
         para_info         = hp)
-      if(h_obj.distance is not None and h_obj.distance > 0.0014):
+      if(h_obj.distance is not None and h_obj.distance > 0.03):
         print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
           residue, 'chain', atoms_list[ih].chain_id, 'distance:', h_obj.distance
       sites_cart[ih] = h_obj.rH_gen
   xray_structure.set_sites_cart(sites_cart)
   pdb_hierarchy.adopt_xray_structure(xray_structure)
-  print pdb_filename
-  STOP()
-  pdb_hierarchy.write_pdb_file(
-    file_name        = pdb_filename+"newH.pdb",
-    crystal_symmetry = xray_structure.crystal_symmetry())
+
+  if pdb_filename is not None:
+    pdb_basename = os.path.basename(pdb_filename.split(".")[0])
+    pdb_hierarchy.write_pdb_file(
+      file_name        = pdb_basename+"_H.pdb",
+      crystal_symmetry = xray_structure.crystal_symmetry())
   print >>log, '*'*79
 
   #for ih in h_parameterization.keys():

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25559
          http://sourceforge.net/p/cctbx/code/25559
Author:   dcliebschner
Date:     2016-10-05 23:30:29 +0000 (Wed, 05 Oct 2016)
Log Message:
-----------
replaced test tst_hydrogen_connectivity.py with tst_h_connectivity.py

Removed Paths:
-------------
    trunk/mmtbx/hydrogens/tst_hydrogen_connectivity.py

Deleted: trunk/mmtbx/hydrogens/tst_hydrogen_connectivity.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_hydrogen_connectivity.py	2016-10-05 23:25:27 UTC (rev 25558)
+++ trunk/mmtbx/hydrogens/tst_hydrogen_connectivity.py	2016-10-05 23:30:29 UTC (rev 25559)
@@ -1,150 +0,0 @@
-from __future__ import division
-import time
-
-import mmtbx.monomer_library.server
-import mmtbx.monomer_library.pdb_interpretation
-import mmtbx.model
-from mmtbx import monomer_library
-from cctbx import geometry_restraints
-import hydrogen_connectivity
-#from mmtbx import hydrogens
-
-pdb_str = """\
-CRYST1   17.955   13.272   13.095  90.00  90.00  90.00 P 1
-ATOM      1  N   TYR A 139      10.241   7.920   5.000  1.00 10.00           N
-ATOM      2  CA  TYR A 139      10.853   7.555   6.271  1.00 10.00           C
-ATOM      3  C   TYR A 139      12.362   7.771   6.227  1.00 10.00           C
-ATOM      4  O   TYR A 139      12.955   8.272   7.181  1.00 10.00           O
-ATOM      5  CB  TYR A 139      10.540   6.098   6.617  1.00 10.00           C
-ATOM      6  CG  TYR A 139       9.063   5.805   6.749  1.00 10.00           C
-ATOM      7  CD1 TYR A 139       8.316   5.391   5.654  1.00 10.00           C
-ATOM      8  CD2 TYR A 139       8.414   5.943   7.969  1.00 10.00           C
-ATOM      9  CE1 TYR A 139       6.966   5.122   5.770  1.00 10.00           C
-ATOM     10  CE2 TYR A 139       7.064   5.676   8.095  1.00 10.00           C
-ATOM     11  CZ  TYR A 139       6.345   5.266   6.993  1.00 10.00           C
-ATOM     12  OH  TYR A 139       5.000   5.000   7.113  1.00 10.00           O
-ATOM     13  HA  TYR A 139      10.480   8.127   6.960  1.00 10.00           H
-ATOM     14  HB2 TYR A 139      10.915   5.524   5.931  1.00 10.00           H
-ATOM     15  HB3 TYR A 139      10.982   5.870   7.450  1.00 10.00           H
-ATOM     16  HD1 TYR A 139       8.732   5.293   4.828  1.00 10.00           H
-ATOM     17  HD2 TYR A 139       8.896   6.220   8.714  1.00 10.00           H
-ATOM     18  HE1 TYR A 139       6.479   4.845   5.028  1.00 10.00           H
-ATOM     19  HE2 TYR A 139       6.643   5.772   8.919  1.00 10.00           H
-ATOM     20  HH  TYR A 139       4.759   5.128   7.907  1.00 10.00           H
-TER
-END
-"""
-
-#----------------------------------------------------
-# This test checks for residue Tyr (pdb_str above):
-# - if all bonds involving H atoms are recognized
-# - if all angles involving H atoms are recognized
-# - if 3rd neighbors of HH atom are correctly found
-#----------------------------------------------------
-
-def exercise():
-  mon_lib_srv = monomer_library.server.server()
-  ener_lib = monomer_library.server.ener_lib()
-  processed_pdb_file = monomer_library.pdb_interpretation.process(
-    mon_lib_srv    = mon_lib_srv,
-    ener_lib       = ener_lib,
-    file_name      = None,
-    raw_records    = pdb_str,
-    force_symmetry = True)
-  pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy
-  xray_structure = processed_pdb_file.xray_structure()
-
-  geometry_restraints = processed_pdb_file.geometry_restraints_manager(
-    show_energies = False)
-  restraints_manager = mmtbx.restraints.manager(
-    geometry      = geometry_restraints,
-    normalization = False)
-
-  sites_cart = xray_structure.sites_cart()
-
-  bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
-      sites_cart = sites_cart)
-  angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
-  hd_selection = xray_structure.hd_selection()
-
-  # determine values with code to be tested
-  connectivity = hydrogen_connectivity.determine_H_neighbors(
-    geometry_restraints   = geometry_restraints,
-    bond_proxies          = bond_proxies_simple,
-    angle_proxies         = angle_proxies,
-    hd_selection          = hd_selection,
-    sites_cart            = sites_cart)
-
-  bond_list = {}
-  angle_list = {}
-  third_nb_list = {}
-  for ih in connectivity.keys():
-    a0 = (connectivity[ih][0])
-    bond_list[ih]=[a0.iseq, a0.dist_ideal]
-    for atom in connectivity[ih][1]:
-      helper = tuple(sorted([ih, a0.iseq, atom.iseq]))
-      angle_list[helper]=atom.angle_ideal
-    if(len(connectivity[ih])==3):
-      third_nb_list[ih]=[]
-      for atom in connectivity[ih][2]:
-        third_nb_list[ih].append(atom.iseq)
-#      print third_nb_list[ih]
-
-#-----------------------------------------------------------------------------
-# This is useful to keep for debugging: human readable output of connectivity
-#-----------------------------------------------------------------------------
-#  for ih in connectivity.keys():
-#    if(len(connectivity[ih])==3):
-#      string = (" ".join([names[p.iseq] for p in connectivity[ih][2]]))
-#    else:
-#      string = 'n/a'
-#    print  names[ih],': ', names[(connectivity[ih][0][0]).iseq], \
-#      ',', (" ".join([names[p.iseq] for p in connectivity[ih][1]])), \
-#      ',', string
-#-----------------------------------------------------------------------------
-
-# determine values directly from pdb_str
-  model = mmtbx.model.manager(
-    restraints_manager = restraints_manager,
-    xray_structure     = xray_structure,
-    pdb_hierarchy      = pdb_hierarchy)
-  bond_ctrl = {}
-  for i in model.xh_connectivity_table():
-    bond_ctrl[i[1]]=[i[0],i[3]]
-
-# We know the angles beforehand
-  angles = [
-    (4, 1, 12),
-    (2, 1, 12),
-    (0, 1, 12),
-    (13, 4, 14),
-    (5, 4, 14),
-    (5, 4, 13),
-    (1, 4, 13),
-    (1, 4, 14),
-    (8, 6, 15),
-    (5, 6, 15),
-    (9, 7, 16),
-    (5, 7, 16),
-    (10, 8, 17),
-    (6, 8, 17),
-    (10, 11, 19),
-    (7, 9, 18),
-    (10, 9, 18)]
-
-  angle_ctrl = {}
-  for ap in angle_proxies:
-    if(ap.i_seqs in angles):
-      angle_ctrl[tuple(sorted(list(ap.i_seqs)))]=ap.angle_ideal
-
-# HH needs also third neighbors:
-  third_nb_ctrl = {19: [8, 9]}
-
-  assert (bond_list == bond_ctrl), '1-2 neighbors and distance_ideal are wrong'
-  assert (angle_list == angle_ctrl), '1-3 neighbors and angle_ideal are wrong'
-  assert (third_nb_list == third_nb_ctrl), '1-4 neighbors are wrong'
-
-if (__name__ == "__main__"):
-  t0 = time.time()
-  exercise()
-  print "OK. Time: %8.3f"%(time.time()-t0)

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25558
          http://sourceforge.net/p/cctbx/code/25558
Author:   dcliebschner
Date:     2016-10-05 23:25:27 +0000 (Wed, 05 Oct 2016)
Log Message:
-----------
test for obtaining dihedral angle in planar XH2 groups

Added Paths:
-----------
    trunk/mmtbx/hydrogens/tst_h_parametrization_3.py

Added: trunk/mmtbx/hydrogens/tst_h_parametrization_3.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_h_parametrization_3.py	                        (rev 0)
+++ trunk/mmtbx/hydrogens/tst_h_parametrization_3.py	2016-10-05 23:25:27 UTC (rev 25558)
@@ -0,0 +1,232 @@
+from __future__ import division
+import time
+import mmtbx.monomer_library.server
+import mmtbx.monomer_library.pdb_interpretation
+from mmtbx import monomer_library
+from cctbx import geometry_restraints
+import hydrogen_connectivity
+import hydrogen_parametrization
+
+#-----------------------------------------------------------------------------
+# This test checks the parameterization of hydrogen atoms for planar X-H2 groups
+# Steps:
+# 1) determine parameterization
+# 2) Compare calculated position of H from parameterization to input position
+# test fails if distance is > 0.02 A
+# Aim is to test if ideal dihedral angles are correct for any configuration
+#-----------------------------------------------------------------------------
+
+def exercise(pdb_str, idealize):
+  mon_lib_srv = monomer_library.server.server()
+  ener_lib = monomer_library.server.ener_lib()
+  processed_pdb_file = monomer_library.pdb_interpretation.process(
+    mon_lib_srv    = mon_lib_srv,
+    ener_lib       = ener_lib,
+    file_name      = None,
+    raw_records    = pdb_str,
+    force_symmetry = True)
+  pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy
+  xray_structure = processed_pdb_file.xray_structure()
+
+  geometry_restraints = processed_pdb_file.geometry_restraints_manager(
+    show_energies = False)
+  restraints_manager = mmtbx.restraints.manager(
+    geometry      = geometry_restraints,
+    normalization = False)
+
+  sites_cart = xray_structure.sites_cart()
+  names = list(pdb_hierarchy.atoms().extract_name())
+  atoms_list = list(pdb_hierarchy.atoms_with_labels())
+
+  bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
+      sites_cart = sites_cart)
+  angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
+  dihedral_proxies = restraints_manager.geometry.dihedral_proxies
+  hd_selection = xray_structure.hd_selection()
+
+  connectivity = hydrogen_connectivity.determine_H_neighbors(
+    geometry_restraints   = geometry_restraints,
+    bond_proxies          = bond_proxies_simple,
+    angle_proxies         = angle_proxies,
+    dihedral_proxies      = dihedral_proxies,
+    hd_selection          = hd_selection,
+    sites_cart            = sites_cart)
+  h_parameterization = hydrogen_parametrization.get_h_parameterization(
+    connectivity   = connectivity,
+    sites_cart     = sites_cart,
+    idealize       = idealize)
+
+  for ih in h_parameterization.keys():
+    residue = atoms_list[ih].resseq
+    hp = h_parameterization[ih]
+    h_obj = hydrogen_parametrization.generate_H_positions(
+      sites_cart = sites_cart,
+      ih         = ih,
+      para_info  = hp)
+    if(h_obj.distance > 0.02):
+      print  hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
+        residue, 'distance:', h_obj.distance
+
+
+pdb_str_00 = """
+CRYST1   13.889   13.738   13.126  90.00  90.00  90.00 P 1
+SCALE1      0.071999  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.072791  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.076185        0.00000
+ATOM      1  N   ASN A   1       5.264   6.323   6.229  1.00 10.00           N
+ATOM      2  CA  ASN A   1       6.108   7.341   6.843  1.00 10.00           C
+ATOM      3  C   ASN A   1       5.944   8.680   6.132  1.00 10.00           C
+ATOM      4  O   ASN A   1       5.516   8.733   4.979  1.00 10.00           O
+ATOM      5  CB  ASN A   1       7.575   6.907   6.817  1.00 10.00           C
+ATOM      6  CG  ASN A   1       7.857   5.750   7.754  1.00 10.00           C
+ATOM      7  OD1 ASN A   1       7.361   5.713   8.880  1.00 10.00           O
+ATOM      8  ND2 ASN A   1       8.659   4.797   7.294  1.00 10.00           N
+ATOM      9  HA  ASN A   1       5.844   7.457   7.769  1.00 10.00           H
+ATOM     10  HB2 ASN A   1       7.806   6.628   5.917  1.00 10.00           H
+ATOM     11  HB3 ASN A   1       8.131   7.655   7.086  1.00 10.00           H
+ATOM     12 HD21 ASN A   1       8.849   4.119   7.787  1.00 10.00           H
+ATOM     13 HD22 ASN A   1       8.988   4.857   6.501  1.00 10.00           H
+"""
+
+pdb_str_01 = """
+CRYST1   13.889   13.738   13.126  90.00  90.00  90.00 P 1
+SCALE1      0.071999  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.072791  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.076185        0.00000
+ATOM      1  N   ASN A   1       5.264   6.323   6.229  1.00 10.00           N
+ATOM      2  CA  ASN A   1       6.108   7.341   6.843  1.00 10.00           C
+ATOM      3  C   ASN A   1       5.944   8.680   6.132  1.00 10.00           C
+ATOM      4  O   ASN A   1       5.516   8.733   4.979  1.00 10.00           O
+ATOM      5  CB  ASN A   1       7.575   6.907   6.817  1.00 10.00           C
+ATOM      6  CG  ASN A   1       7.857   5.750   7.754  1.00 10.00           C
+ATOM      7  OD1 ASN A   1       7.361   5.713   8.880  1.00 10.00           O
+ATOM      8  ND2 ASN A   1       8.659   4.797   7.294  1.00 10.00           N
+ATOM      9  HA  ASN A   1       5.844   7.457   7.769  1.00 10.00           H
+ATOM     10  HB2 ASN A   1       7.806   6.628   5.917  1.00 10.00           H
+ATOM     11  HB3 ASN A   1       8.131   7.655   7.086  1.00 10.00           H
+ATOM     12 HD21 ASN A   1       8.988   4.857   6.501  1.00 10.00           H
+ATOM     13 HD22 ASN A   1       8.849   4.119   7.787  1.00 10.00           H
+"""
+
+pdb_str_02 = """
+CRYST1   14.446   16.451   11.913  90.00  90.00  90.00 P 1
+SCALE1      0.069223  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.060787  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.083942        0.00000
+ATOM      1  N   ARG A  50       6.725  11.631   6.607  1.00 10.00           N
+ATOM      2  CA  ARG A  50       7.490  10.491   6.117  1.00 10.00           C
+ATOM      3  C   ARG A  50       8.914  10.908   5.763  1.00 10.00           C
+ATOM      4  O   ARG A  50       9.701  11.267   6.639  1.00 10.00           O
+ATOM      5  CB  ARG A  50       6.806   9.870   4.897  1.00 10.00           C
+ATOM      6  CG  ARG A  50       5.402   9.344   5.166  1.00 10.00           C
+ATOM      7  CD  ARG A  50       5.419   8.122   6.072  1.00 10.00           C
+ATOM      8  NE  ARG A  50       6.111   6.992   5.457  1.00 10.00           N
+ATOM      9  CZ  ARG A  50       6.529   5.916   6.119  1.00 10.00           C
+ATOM     10  NH1 ARG A  50       6.327   5.812   7.427  1.00 10.00           N
+ATOM     11  NH2 ARG A  50       7.150   4.940   5.472  1.00 10.00           N
+ATOM     12  HA  ARG A  50       7.540   9.818   6.814  1.00 10.00           H
+ATOM     13  HB2 ARG A  50       6.740  10.543   4.202  1.00 10.00           H
+ATOM     14  HB3 ARG A  50       7.345   9.127   4.582  1.00 10.00           H
+ATOM     15  HG2 ARG A  50       4.880  10.036   5.600  1.00 10.00           H
+ATOM     16  HG3 ARG A  50       4.990   9.092   4.325  1.00 10.00           H
+ATOM     17  HD2 ARG A  50       5.872   8.343   6.900  1.00 10.00           H
+ATOM     18  HD3 ARG A  50       4.505   7.851   6.254  1.00 10.00           H
+ATOM     19  HE  ARG A  50       6.259   7.024   4.611  1.00 10.00           H
+ATOM     20 HH11 ARG A  50       5.925   6.440   7.855  1.00 10.00           H
+ATOM     21 HH12 ARG A  50       6.600   5.114   7.847  1.00 10.00           H
+ATOM     22 HH21 ARG A  50       7.283   5.003   4.624  1.00 10.00           H
+ATOM     23 HH22 ARG A  50       7.420   4.245   5.899  1.00 10.00           H
+"""
+
+pdb_str_03 = """
+CRYST1   14.446   16.451   11.913  90.00  90.00  90.00 P 1
+SCALE1      0.069223  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.060787  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.083942        0.00000
+ATOM      1  N   ARG A  50       6.725  11.631   6.607  1.00 10.00           N
+ATOM      2  CA  ARG A  50       7.490  10.491   6.117  1.00 10.00           C
+ATOM      3  C   ARG A  50       8.914  10.908   5.763  1.00 10.00           C
+ATOM      4  O   ARG A  50       9.701  11.267   6.639  1.00 10.00           O
+ATOM      5  CB  ARG A  50       6.806   9.870   4.897  1.00 10.00           C
+ATOM      6  CG  ARG A  50       5.402   9.344   5.166  1.00 10.00           C
+ATOM      7  CD  ARG A  50       5.419   8.122   6.072  1.00 10.00           C
+ATOM      8  NE  ARG A  50       6.111   6.992   5.457  1.00 10.00           N
+ATOM      9  CZ  ARG A  50       6.529   5.916   6.119  1.00 10.00           C
+ATOM     10  NH1 ARG A  50       6.327   5.812   7.427  1.00 10.00           N
+ATOM     11  NH2 ARG A  50       7.150   4.940   5.472  1.00 10.00           N
+ATOM     12  HA  ARG A  50       7.540   9.818   6.814  1.00 10.00           H
+ATOM     13  HB2 ARG A  50       6.740  10.543   4.202  1.00 10.00           H
+ATOM     14  HB3 ARG A  50       7.345   9.127   4.582  1.00 10.00           H
+ATOM     15  HG2 ARG A  50       4.880  10.036   5.600  1.00 10.00           H
+ATOM     16  HG3 ARG A  50       4.990   9.092   4.325  1.00 10.00           H
+ATOM     17  HD2 ARG A  50       5.872   8.343   6.900  1.00 10.00           H
+ATOM     18  HD3 ARG A  50       4.505   7.851   6.254  1.00 10.00           H
+ATOM     19  HE  ARG A  50       6.259   7.024   4.611  1.00 10.00           H
+ATOM     21 HH11 ARG A  50       6.600   5.114   7.847  1.00 10.00           H
+ATOM     20 HH12 ARG A  50       5.925   6.440   7.855  1.00 10.00           H
+ATOM     23 HH21 ARG A  50       7.420   4.245   5.899  1.00 10.00           H
+ATOM     22 HH22 ARG A  50       7.283   5.003   4.624  1.00 10.00           H
+"""
+
+pdb_str_04 = """
+CRYST1   14.547   13.375   15.374  90.00  90.00  90.00 P 1
+SCALE1      0.068743  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.074766  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.065045        0.00000
+ATOM      1  N   GLN A  70       6.510   6.937   4.976  1.00 10.00           N
+ATOM      2  CA  GLN A  70       6.727   6.312   6.275  1.00 10.00           C
+ATOM      3  C   GLN A  70       5.406   6.141   7.018  1.00 10.00           C
+ATOM      4  O   GLN A  70       4.992   5.022   7.317  1.00 10.00           O
+ATOM      5  CB  GLN A  70       7.700   7.144   7.113  1.00 10.00           C
+ATOM      6  CG  GLN A  70       8.079   6.506   8.440  1.00 10.00           C
+ATOM      7  CD  GLN A  70       9.104   7.319   9.206  1.00 10.00           C
+ATOM      8  OE1 GLN A  70       9.517   8.392   8.767  1.00 10.00           O
+ATOM      9  NE2 GLN A  70       9.521   6.810  10.360  1.00 10.00           N
+ATOM     10  HA  GLN A  70       7.116   5.433   6.144  1.00 10.00           H
+ATOM     11  HB2 GLN A  70       8.516   7.275   6.605  1.00 10.00           H
+ATOM     12  HB3 GLN A  70       7.292   8.003   7.304  1.00 10.00           H
+ATOM     13  HG2 GLN A  70       7.287   6.427   8.993  1.00 10.00           H
+ATOM     14  HG3 GLN A  70       8.456   5.628   8.273  1.00 10.00           H
+ATOM     15 HE21 GLN A  70       9.210   6.057  10.635  1.00 10.00           H
+ATOM     16 HE22 GLN A  70      10.103   7.233  10.832  1.00 10.00           H
+"""
+
+pdb_str_05 = """
+CRYST1   14.547   13.375   15.374  90.00  90.00  90.00 P 1
+SCALE1      0.068743  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.074766  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.065045        0.00000
+ATOM      1  N   GLN A  70       6.510   6.937   4.976  1.00 10.00           N
+ATOM      2  CA  GLN A  70       6.727   6.312   6.275  1.00 10.00           C
+ATOM      3  C   GLN A  70       5.406   6.141   7.018  1.00 10.00           C
+ATOM      4  O   GLN A  70       4.992   5.022   7.317  1.00 10.00           O
+ATOM      5  CB  GLN A  70       7.700   7.144   7.113  1.00 10.00           C
+ATOM      6  CG  GLN A  70       8.079   6.506   8.440  1.00 10.00           C
+ATOM      7  CD  GLN A  70       9.104   7.319   9.206  1.00 10.00           C
+ATOM      8  OE1 GLN A  70       9.517   8.392   8.767  1.00 10.00           O
+ATOM      9  NE2 GLN A  70       9.521   6.810  10.360  1.00 10.00           N
+ATOM     10  HA  GLN A  70       7.116   5.433   6.144  1.00 10.00           H
+ATOM     11  HB2 GLN A  70       8.516   7.275   6.605  1.00 10.00           H
+ATOM     12  HB3 GLN A  70       7.292   8.003   7.304  1.00 10.00           H
+ATOM     13  HG2 GLN A  70       7.287   6.427   8.993  1.00 10.00           H
+ATOM     14  HG3 GLN A  70       8.456   5.628   8.273  1.00 10.00           H
+ATOM     16 HE21 GLN A  70      10.103   7.233  10.832  1.00 10.00           H
+ATOM     15 HE22 GLN A  70       9.210   6.057  10.635  1.00 10.00           H
+"""
+
+pdb_list = [pdb_str_00, pdb_str_01, pdb_str_02, pdb_str_03,
+  pdb_str_04, pdb_str_05]
+
+pdb_list_name = ['pdb_str_00', 'pdb_str_01', 'pdb_str_02', 'pdb_str_03',
+  'pdb_str_04', 'pdb_str_05']
+
+def run():
+  #for idealize in [True, False]:
+  for pdb_str, str_name in zip(pdb_list,pdb_list_name):
+    print 'pdb_string:', str_name, 'idealize =', True
+    exercise(pdb_str=pdb_str, idealize=True)
+
+if (__name__ == "__main__"):
+  t0 = time.time()
+  run()
+  print "OK. Time: %8.3f"%(time.time()-t0)

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25557
          http://sourceforge.net/p/cctbx/code/25557
Author:   dcliebschner
Date:     2016-10-05 23:21:00 +0000 (Wed, 05 Oct 2016)
Log Message:
-----------
code for modification of gradients

Added Paths:
-----------
    trunk/mmtbx/hydrogens/modify_gradients.py
    trunk/mmtbx/hydrogens/tst_run_fd_1.py
    trunk/mmtbx/hydrogens/tst_run_fd_2.py
    trunk/mmtbx/hydrogens/tst_run_fd_3.py
    trunk/mmtbx/hydrogens/tst_run_fd_4.py

Added: trunk/mmtbx/hydrogens/modify_gradients.py
===================================================================
--- trunk/mmtbx/hydrogens/modify_gradients.py	                        (rev 0)
+++ trunk/mmtbx/hydrogens/modify_gradients.py	2016-10-05 23:21:00 UTC (rev 25557)
@@ -0,0 +1,206 @@
+from __future__ import division
+from libtbx import adopt_init_args
+from scitbx import matrix
+import math
+
+
+
+# transformation for derivative of a unit vector
+def G_unitvector(Gu, r):
+  x, y, z = r[0], r[1], r[2]
+  Gxu, Gyu, Gzu = Gu[0], Gu[1], Gu[2]
+  #denom = math.sqrt((x**2 + y**2 + z**2)**3)
+  denom = (r.length())**3
+  if(denom == 0):
+    raise RuntimeError("division by zero in G_unitvector")
+  Gx = ( (y**2 + z**2)*Gu[0] - x*y*Gu[1] - x*z*Gu[2] ) / denom
+  Gy = ( -x*y*Gu[0] + (x**2 + z**2)*Gu[1] - y*z*Gu[2] ) / denom
+  Gz = ( -x*z*Gu[0] - y*z*Gu[1] + (x**2 + y**2)*Gu[2] ) / denom
+  return matrix.col([Gx, Gy, Gz])
+#
+# transformation for cross product
+def G_crossproduct(Gv, r1, r2):
+  x1, y1, z1 = r1
+  x2, y2, z2 = r2
+  Gx_v, Gy_v, Gz_v = Gv
+  Gx1 = -z2 * Gy_v + y2 * Gz_v
+  Gy1 =  z2 * Gx_v - x2 * Gz_v
+  Gz1 = -y2 * Gx_v + x2 * Gy_v
+  Gx2 =  z1 * Gy_v - y1 * Gz_v
+  Gy2 = -z1 * Gx_v + x1 * Gz_v
+  Gz2 =  y1 * Gx_v - x1 * Gy_v
+  Gr1 = matrix.col([Gx1, Gy1, Gz1])
+  Gr2 = matrix.col([Gx2, Gy2, Gz2])
+  return Gr1, Gr2
+#
+
+class modify_gradients(object):
+
+  def __init__(self,
+               sites_cart,
+               h_parameterization,
+               grads):
+    adopt_init_args(self, locals())
+
+    for ih in self.h_parameterization.keys():
+      hp = self.h_parameterization[ih]
+      if(hp.htype == 'unk'):
+        continue
+      a0, a1, a2 = hp.a0, hp.a1, hp.a2
+      a, b, h  = hp.a, hp.b, hp.h
+      dh = hp.dist_h
+      rh = matrix.col(sites_cart[ih])
+      r0 = matrix.col(sites_cart[a0])
+      GH = matrix.col(self.grads[ih])
+      G0 = matrix.col(self.grads[a0])
+      G1 = matrix.col(self.grads[a1])
+      G2 = matrix.col(self.grads[a2])
+      if (hp.htype in ['flat_2neigbs', '2neigbs', '2tetra']):
+        r1 = matrix.col(sites_cart[a1])
+        r2 = matrix.col(sites_cart[a2])
+        r10 = r1 - r0
+        r20 = r2 - r0
+        u10 = (r1 - r0).normalize()
+        u20 = (r2 - r0).normalize()
+      # step 1
+        GuH0 = dh * GH
+        G01 = GH
+      # step 2
+        if(hp.htype == 'flat_2neigbs'):
+          rh0 = (a * u10 + b * u20)
+          # alternatively:
+          #length = math.sqrt(rh0.dot(rh0))
+          length = math.sqrt(a*a + b*b + 2*a*b*(u10).dot(u20))
+          Gu10 = a/length*GuH0-a*b*u20*(1/length)**3*rh0.dot(GuH0)
+          Gu20 = b/length*GuH0-a*b*u10*(1/length)**3*rh0.dot(GuH0)
+        elif (hp.htype == '2tetra'):
+          alpha = hp.alpha
+          v = u10.cross(u20)
+          d = a * u10 + b * u20
+        # step 2
+          Gd0 = math.cos(alpha) * GuH0
+          Gv0 = math.sin(alpha) * GuH0
+        # step 3
+          Gd = G_unitvector(Gu = Gd0, r  = d)
+          Gv = G_unitvector(Gu = Gv0, r  = v)
+        # step 4
+          Gu10_1 = a * Gd
+          Gu20_1 = b * Gd
+        # step 5
+          Gu10_2, Gu20_2 = G_crossproduct(Gv, u10, u20)
+          Gu10 = Gu10_1 + Gu10_2
+          Gu20 = Gu20_1 + Gu20_2
+        elif (hp.htype == '2neigbs'):
+          v = u10.cross(u20)
+          vl = v.length()
+          v0 = v.normalize()
+          rh0 = (a * u10 + b * u20 + h * v0)
+          length = math.sqrt(rh0.dot(rh0))
+          # alternatively:
+          #length = math.sqrt(a*a + b*b + h*h + 2*a*b*(u10).dot(u20))
+          Gu10_1 = a/length*GuH0 - (a*b*u20)*(1/length)**3*rh0.dot(GuH0)
+          Gu20_1 = b/length*GuH0 - (a*b*u10)*(1/length)**3*rh0.dot(GuH0)
+          Gv0 = h/length*GuH0
+      # step 3
+          Gv = G_unitvector(Gu = Gv0, r  = v)
+      # step 4
+          Gu10_2, Gu20_2 = G_crossproduct(Gv, u10, u20)
+          Gu10 = Gu10_1 + Gu10_2
+          Gu20 = Gu20_1 + Gu20_2
+        # step 5
+        Gr10 = G_unitvector(Gu = Gu10, r  = r10)
+        Gr20 = G_unitvector(Gu = Gu20, r  = r20)
+        #
+        self.grads[a0] = (G0[0] + GH[0] - Gr10[0] - Gr20[0],
+                          G0[1] + GH[1] - Gr10[1] - Gr20[1],
+                          G0[2] + GH[2] - Gr10[2] - Gr20[2])
+        self.grads[a1] = (G1[0] + Gr10[0],
+                          G1[1] + Gr10[1],
+                          G1[2] + Gr10[2])
+        self.grads[a2] = (G2[0] + Gr20[0],
+                          G2[1] + Gr20[1],
+                          G2[2] + Gr20[2])
+      #
+      if (hp.htype == '3neigbs'):
+        a3 = hp.a3
+        G3 = matrix.col(self.grads[a3])
+        r1 = matrix.col(sites_cart[a1])
+        r2 = matrix.col(sites_cart[a2])
+        r3 = matrix.col(sites_cart[a3])
+        r10, r20, r30 = r1 - r0, r2 - r0, r3 - r0
+        u10 = r10.normalize()
+        u20 = r20.normalize()
+        u30 = r30.normalize()
+        rh0 = (a * u10 + b * u20 + h * u30)
+        # alternatively:
+        #length = math.sqrt(rh0.dot(rh0))
+        length = math.sqrt(a*a + b*b + h*h+ 2*a*b*(u10).dot(u20)
+          + 2*a*h*(u10).dot(u30) + 2*b*h*(u20).dot(u30))
+      # step 1
+        GuH0 = dh * GH
+        G01 = GH
+      # step 2
+        Gu10 = a/length*GuH0 - (a*b*u20+a*h*u30)*(1/length)**3*rh0.dot(GuH0)
+        Gu20 = b/length*GuH0 - (a*b*u10+b*h*u30)*(1/length)**3*rh0.dot(GuH0)
+        Gu30 = h/length*GuH0 - (a*h*u10+b*h*u20)*(1/length)**3*rh0.dot(GuH0)
+      # step 3
+        Gr10 = G_unitvector(Gu = Gu10, r  = r10)
+        Gr20 = G_unitvector(Gu = Gu20, r  = r20)
+        Gr30 = G_unitvector(Gu = Gu30, r  = r30)
+        #
+        self.grads[a0] = (G0[0] + GH[0] - Gr10[0] - Gr20[0] - Gr30[0],
+                          G0[1] + GH[1] - Gr10[1] - Gr20[1] - Gr30[1],
+                          G0[2] + GH[2] - Gr10[2] - Gr20[2] - Gr30[2])
+        self.grads[a1] = (G1[0] + Gr10[0],
+                          G1[1] + Gr10[1],
+                          G1[2] + Gr10[2])
+        self.grads[a2] = (G2[0] + Gr20[0],
+                          G2[1] + Gr20[1],
+                          G2[2] + Gr20[2])
+        self.grads[a3] = (G3[0] + Gr30[0],
+                          G3[1] + Gr30[1],
+                          G3[2] + Gr30[2])
+
+      if (hp.htype in ['alg1b', 'prop', 'alg1a']):
+        r1 = matrix.col(sites_cart[a1])
+        rb1 = matrix.col(sites_cart[a2])
+        phi, alpha, n = hp.phi, hp.alpha, hp.n
+        k1 = -math.cos(alpha)
+        k2 = math.sin(alpha)*math.cos(phi + n*2*math.pi/3)
+        k3 = math.sin(alpha)*math.sin(phi + n*2*math.pi/3)
+        r01 = r0 - r1
+        rb10 = rb1 - r1
+        v1 = r0 - r1
+        u1 = (r0 - r1).normalize()
+        v2 = rb10 - ((rb10).dot(u1)) * u1
+        u2 = v2.normalize()
+        u3 = u1.cross(u2)
+      # step 1
+        GuH0 = dh * GH
+        G01 = GH
+      # step 2
+        Gu1_1 = k1 * GuH0
+        Gu2_1 = k2 * GuH0
+        Gu3   = k3 * GuH0
+      # step 3
+        Gu1_2, Gu2_2 = G_crossproduct(Gu3, u1, u2)
+        Gu2 = Gu2_1 + Gu2_2
+      # step 4
+        Gv2 = G_unitvector(Gu = Gu2, r  = v2)
+      # step 5
+        Gu1 = Gu1_1 + Gu1_2 - (rb10.dot(u1))*Gv2 - (u1.dot(Gv2))*rb10
+        Grb1 = Gv2 - (u1.dot(Gv2))*u1
+      # step 6
+        Gr01 = G_unitvector(Gu = Gu1, r  = r01)
+      #
+        self.grads[a0] = (G0[0] + GH[0] + Gr01[0],
+                          G0[1] + GH[1] + Gr01[1],
+                          G0[2] + GH[2] + Gr01[2])
+        self.grads[a1] = (G1[0] - Gr01[0] - Grb1[0],
+                          G1[1] - Gr01[1] - Grb1[1],
+                          G1[2] - Gr01[2] - Grb1[2])
+        self.grads[a2] = (G2[0] + Grb1[0],
+                          G2[1] + Grb1[1],
+                          G2[2] + Grb1[2])
+      #
+      self.grads[ih] = (0,0,0)

Added: trunk/mmtbx/hydrogens/tst_run_fd_1.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_run_fd_1.py	                        (rev 0)
+++ trunk/mmtbx/hydrogens/tst_run_fd_1.py	2016-10-05 23:21:00 UTC (rev 25557)
@@ -0,0 +1,315 @@
+from __future__ import division
+import time
+import sys
+from cctbx.array_family import flex
+from mmtbx import monomer_library
+import mmtbx.monomer_library.server
+import mmtbx.monomer_library.pdb_interpretation
+from libtbx.test_utils import approx_equal
+import hydrogen_connectivity
+import hydrogen_parametrization
+import modify_gradients
+from scitbx import matrix
+
+#-----------------------------------------------------------------------------
+# Finite difference test for modified gradients of riding H
+# Every type of riding H is tested:
+# planar H
+# tetragonal H
+# two tetragonal H
+# X-H2 group (ARG etc)
+# propeller group
+# H with rotational degree of freedom (SER, etc)
+#-----------------------------------------------------------------------------
+
+def exercise(pdb_str, eps, idealize):
+  mon_lib_srv = monomer_library.server.server()
+  ener_lib = monomer_library.server.ener_lib()
+  processed_pdb_file = monomer_library.pdb_interpretation.process(
+      mon_lib_srv    = mon_lib_srv,
+      ener_lib       = ener_lib,
+      raw_records    = pdb_str,
+      force_symmetry = True)
+  xray_structure = processed_pdb_file.xray_structure()
+  geometry = processed_pdb_file.geometry_restraints_manager(
+      show_energies      = False,
+      plain_pairs_radius = 5.0)
+  sites_cart = xray_structure.sites_cart()
+  es = geometry.energies_sites(
+    sites_cart = sites_cart,
+    compute_gradients = True)
+  g_analytical = es.gradients
+#
+  pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy
+  restraints_manager = mmtbx.restraints.manager(
+    geometry      = geometry,
+    normalization = False)
+  bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
+    sites_cart = xray_structure.sites_cart())
+  angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
+  dihedral_proxies = restraints_manager.geometry.dihedral_proxies
+  names = list(pdb_hierarchy.atoms().extract_name())
+  atoms_list = list(pdb_hierarchy.atoms_with_labels())
+  hd_selection = xray_structure.hd_selection()
+  connectivity = hydrogen_connectivity.determine_H_neighbors(
+    geometry_restraints   = geometry,
+    bond_proxies          = bond_proxies_simple,
+    angle_proxies         = angle_proxies,
+    dihedral_proxies      = dihedral_proxies,
+    hd_selection          = hd_selection,
+    sites_cart            = sites_cart)
+  h_parameterization = hydrogen_parametrization.get_h_parameterization(
+    connectivity   = connectivity,
+    sites_cart     = sites_cart,
+    idealize       = idealize)
+
+  ih = h_parameterization.keys()[0]
+  h_obj = hydrogen_parametrization.generate_H_positions(
+    sites_cart = sites_cart,
+    ih         = ih,
+    para_info  = h_parameterization[ih])
+  print 'distance before changing H', (h_obj.rH_gen - matrix.col(sites_cart[ih])).length()
+
+  sites_cart[ih] = tuple(h_obj.rH_gen)
+  xray_structure.set_sites_cart(sites_cart)
+  h_obj = hydrogen_parametrization.generate_H_positions(
+    sites_cart = sites_cart,
+    ih         = ih,
+    para_info  = h_parameterization[ih])
+  print 'distance H_calc - H_mod', (h_obj.rH_gen - matrix.col(sites_cart[ih])).length()
+
+  #for i in g_analytical:
+  #  print i
+  #print '----------'
+  g_analytical = geometry.energies_sites(
+    sites_cart = sites_cart,
+    compute_gradients = True).gradients
+  modify_gradients.modify_gradients(
+    sites_cart         = sites_cart,
+    h_parameterization = h_parameterization,
+    grads              = g_analytical)
+   #
+  ex = [eps,0,0]
+  ey = [0,eps,0]
+  ez = [0,0,eps]
+  g_fd = flex.vec3_double()
+  for i_site in xrange(sites_cart.size()):
+    g_fd_i = []
+    for e in [ex,ey,ez]:
+      ts = []
+      for sign in [-1,1]:
+        sites_cart_ = sites_cart.deep_copy()
+        sites_cart_[i_site] = [
+          sites_cart_[i_site][j]+e[j]*sign for j in xrange(3)]
+        # after shift, recalculate H position
+        for ih in h_parameterization.keys():
+          h_obj = hydrogen_parametrization.generate_H_positions(
+            sites_cart = sites_cart_,
+            ih         = ih,
+            para_info  = h_parameterization[ih])
+          sites_cart_[ih] = tuple(h_obj.rH_gen)
+        ts.append(geometry.energies_sites(
+          sites_cart = sites_cart_,
+          compute_gradients = False).target)
+      g_fd_i.append((ts[1]-ts[0])/(2*eps))
+    g_fd.append(g_fd_i)
+  #
+  for g1, g2 in zip(g_analytical, g_fd):
+    print g1,g2
+    assert approx_equal(g1,g2, 1.e-4)
+  print '*'*79
+
+def print_para(h_parameterization, ih):
+  for ih in h_parameterization.keys():
+    hp = h_parameterization[ih]
+    print hp.htype, 'ih', ih, ', a0 ', hp.a0, ', a1 ', hp.a1, 'a2 = ', hp.a2, \
+      'a = ', hp.a, 'b = ', hp.b, 'h = ', hp.h, 'dist_h =', hp.dist_h
+
+# alg2a shaked
+pdb_str_00 = """
+CRYST1   16.660   13.261   16.215  90.00  90.00  90.00 P 1
+SCALE1      0.060024  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075409  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.061671        0.00000
+ATOM      6  CG  TYR A   7       8.528   6.445   8.572  1.00 15.00           C
+ATOM      8  CD2 TYR A   7       8.348   7.014   9.545  1.00 15.00           C
+ATOM     10  CE2 TYR A   7       7.068   7.273  10.230  1.00 15.00           C
+ATOM     17  HD2 TYR A   7       9.064   7.637  10.081  1.00 15.00           H
+TER
+"""
+# alg2a ideal geometry
+pdb_str_01 = """
+CRYST1   16.660   13.261   16.215  90.00  90.00  90.00 P 1
+SCALE1      0.060024  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075409  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.061671        0.00000
+ATOM      1  CG  TYR A   7       8.544   6.391   8.506  1.00 15.00           C
+ATOM      2  CD2 TYR A   7       8.334   7.144   9.654  1.00 15.00           C
+ATOM      3  CE2 TYR A   7       7.078   7.251  10.220  1.00 15.00           C
+ATOM      4  HD2 TYR A   7       9.052   7.584  10.049  1.00 15.00           H
+TER
+"""
+# alg2b shaked
+pdb_str_02 = """
+CRYST1   16.660   13.261   16.215  90.00  90.00  90.00 P 1
+SCALE1      0.060024  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075409  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.061671        0.00000
+ATOM      2  CA  TYR A   7       9.887   7.460   6.606  1.00 15.00           C
+ATOM      5  CB  TYR A   7       9.830   6.306   7.775  1.00 15.00           C
+ATOM      6  CG  TYR A   7       8.528   6.445   8.572  1.00 15.00           C
+ATOM     14  HB2 TYR A   7       9.943   5.616   7.588  1.00 15.00           H
+END
+"""
+# alg2b ideal geometry
+pdb_str_03 = """
+CRYST1   16.660   13.261   16.215  90.00  90.00  90.00 P 1
+SCALE1      0.060024  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075409  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.061671        0.00000
+ATOM      2  CA  TYR A   7       9.962   6.614   6.452  1.00 15.00           C
+ATOM      5  CB  TYR A   7       9.920   6.991   7.935  1.00 15.00           C
+ATOM      6  CG  TYR A   7       8.575   6.781   8.602  1.00 15.00           C
+ATOM     14  HB2 TYR A   7      10.571   6.452   8.411  1.00 15.00           H
+"""
+
+# alg3 shaked
+pdb_str_04 = """
+CRYST1   16.660   13.261   16.215  90.00  90.00  90.00 P 1
+SCALE1      0.060024  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075409  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.061671        0.00000
+ATOM      1  N   TYR A   7       9.035   7.089   5.637  1.00 15.00           N
+ATOM      2  CA  TYR A   7       9.887   7.460   6.606  1.00 15.00           C
+ATOM      3  C   TYR A   7      11.285   7.169   6.250  1.00 15.00           C
+ATOM      5  CB  TYR A   7       9.830   6.306   7.775  1.00 15.00           C
+ATOM     13  HA  TYR A   7       9.831   8.340   7.271  1.00 15.00           H
+"""
+
+# alg3 ideal geometry
+pdb_str_05 = """
+CRYST1   16.660   13.261   16.215  90.00  90.00  90.00 P 1
+SCALE1      0.060024  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075409  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.061671        0.00000
+ATOM      1  N   TYR A   7       8.981   7.170   5.658  1.00 15.00           N
+ATOM      2  CA  TYR A   7       9.949   7.360   6.732  1.00 15.00           C
+ATOM      3  C   TYR A   7      11.374   7.278   6.195  1.00 15.00           C
+ATOM      4  CB  TYR A   7       9.740   6.317   7.832  1.00 15.00           C
+ATOM      5  HA  TYR A   7       9.823   8.239   7.122  1.00 15.00           H
+"""
+
+# alg1a shaked
+pdb_str_06 = """
+CRYST1   14.446   16.451   11.913  90.00  90.00  90.00 P 1
+SCALE1      0.069223  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.060787  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.083942        0.00000
+ATOM      8  NE  ARG A  50       5.682   6.869   5.406  1.00 10.00           N
+ATOM      9  CZ  ARG A  50       6.378   6.261   5.888  1.00 10.00           C
+ATOM     10  NH1 ARG A  50       7.214   6.770   6.820  1.00 10.00           N
+ATOM     11  NH2 ARG A  50       6.924   5.048   5.452  1.00 10.00           N
+ATOM     21 HH12 ARG A  50       7.949   6.540   7.393  1.00 10.00           H
+"""
+
+# alg1a idealized
+pdb_str_07 = """
+CRYST1   14.446   16.451   11.913  90.00  90.00  90.00 P 1
+SCALE1      0.069223  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.060787  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.083942        0.00000
+ATOM      1  NE  ARG A  50       5.543   6.877   5.312  1.00 10.00           N
+ATOM      2  CZ  ARG A  50       6.559   6.257   5.905  1.00 10.00           C
+ATOM      3  NH1 ARG A  50       7.213   6.851   6.894  1.00 10.00           N
+ATOM      4  NH2 ARG A  50       6.920   5.044   5.510  1.00 10.00           N
+ATOM      6 HH12 ARG A  50       7.870   6.449   7.277  1.00 10.00           H
+"""
+
+#alg1b_flat shaked
+pdb_str_08 = """
+CRYST1   16.660   13.261   16.215  90.00  90.00  90.00 P 1
+SCALE1      0.060024  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075409  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.061671        0.00000
+ATOM      9  CE1 TYR A   7       6.338   5.396   8.586  1.00 15.00           C
+ATOM     10  CE2 TYR A   7       7.068   7.273  10.230  1.00 15.00           C
+ATOM     11  CZ  TYR A   7       6.090   6.501   9.765  1.00 15.00           C
+ATOM     12  OH  TYR A   7       4.857   6.463  10.553  1.00 15.00           O
+ATOM     20  HH  TYR A   7       5.208   7.016  11.256  1.00 15.00           H
+"""
+
+#alg1b_flat idealized
+pdb_str_09 = """
+CRYST1   16.660   13.261   16.215  90.00  90.00  90.00 P 1
+SCALE1      0.060024  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075409  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.061671        0.00000
+ATOM      1  CE1 TYR A   7       6.239   5.565   8.724  1.00 15.00           C
+ATOM      2  CE2 TYR A   7       7.170   7.259  10.131  1.00 15.00           C
+ATOM      3  CZ  TYR A   7       6.130   6.416   9.803  1.00 15.00           C
+ATOM      4  OH  TYR A   7       4.978   6.424  10.555  1.00 15.00           O
+ATOM      5  HH  TYR A   7       5.045   6.985  11.177  1.00 15.00           H
+"""
+
+#alg1b shaked (HH out of plane)
+pdb_str_10 = """
+CRYST1   16.660   13.261   16.215  90.00  90.00  90.00 P 1
+SCALE1      0.060024  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075409  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.061671        0.00000
+ATOM      9  CE1 TYR A   7       6.231   6.395   8.363  1.00 15.00           C
+ATOM     10  CE2 TYR A   7       7.119   6.748  10.501  1.00 15.00           C
+ATOM     11  CZ  TYR A   7       5.926   6.249   9.983  1.00 15.00           C
+ATOM     12  OH  TYR A   7       4.831   6.083  10.481  1.00 15.00           O
+ATOM     20  HH  TYR A   7       4.708   6.641  11.301  1.00 15.00           H
+"""
+
+# alg1b idealized (HH out of plane)
+pdb_str_11 = """
+CRYST1   16.660   13.261   16.215  90.00  90.00  90.00 P 1
+SCALE1      0.060024  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075409  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.061671        0.00000
+ATOM      9  CE1 TYR A   7       6.193   6.410   8.461  1.00 15.00           C
+ATOM     10  CE2 TYR A   7       7.242   6.572  10.604  1.00 15.00           C
+ATOM     11  CZ  TYR A   7       6.111   6.396   9.837  1.00 15.00           C
+ATOM     12  OH  TYR A   7       4.893   6.206  10.448  1.00 15.00           O
+ATOM     20  HH  TYR A   7       4.878   6.603  11.188  1.00 15.00           H
+"""
+
+#alg1a shaked with both H atoms
+pdb_str_06_bla = """
+CRYST1   14.446   16.451   11.913  90.00  90.00  90.00 P 1
+SCALE1      0.069223  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.060787  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.083942        0.00000
+ATOM      8  NE  ARG A  50       5.682   6.869   5.406  1.00 10.00           N
+ATOM      9  CZ  ARG A  50       6.378   6.261   5.888  1.00 10.00           C
+ATOM     10  NH1 ARG A  50       7.214   6.770   6.820  1.00 10.00           N
+ATOM     11  NH2 ARG A  50       6.924   5.048   5.452  1.00 10.00           N
+ATOM     20 HH11 ARG A  50       6.939   7.628   7.092  1.00 10.00           H
+ATOM     21 HH12 ARG A  50       7.949   6.540   7.393  1.00 10.00           H
+"""
+
+pdb_list = [pdb_str_00, pdb_str_01, pdb_str_02, pdb_str_03,
+  pdb_str_04, pdb_str_05, pdb_str_06, pdb_str_07, pdb_str_08, pdb_str_09,
+  pdb_str_10, pdb_str_11]
+
+pdb_list_name = ['pdb_str_00', 'pdb_str_01', 'pdb_str_02', 'pdb_str_03',
+  'pdb_str_04', 'pdb_str_05', 'pdb_str_06', 'pdb_str_07', 'pdb_str_08',
+  'pdb_str_09', 'pdb_str_10', 'pdb_str_11']
+
+#pdb_list = [pdb_str_02, pdb_str_03]
+#pdb_list_name = ['pdb_str_02', 'pdb_str_03']
+
+def run():
+  for idealize in [True, False]:
+    for pdb_str, str_name in zip(pdb_list,pdb_list_name):
+      print 'pdb_string:', str_name, 'idealize =', idealize
+      exercise(pdb_str=pdb_str, eps=1.e-4, idealize=idealize)
+
+if (__name__ == "__main__"):
+  run()
+  t0 = time.time()
+  #run(sys.argv[1:])
+  print "OK. Time:", round(time.time()-t0, 2), "seconds"

Added: trunk/mmtbx/hydrogens/tst_run_fd_2.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_run_fd_2.py	                        (rev 0)
+++ trunk/mmtbx/hydrogens/tst_run_fd_2.py	2016-10-05 23:21:00 UTC (rev 25557)
@@ -0,0 +1,600 @@
+from __future__ import division
+import time
+import sys
+from cctbx.array_family import flex
+from mmtbx import monomer_library
+import mmtbx.monomer_library.server
+import mmtbx.monomer_library.pdb_interpretation
+from libtbx.test_utils import approx_equal
+import hydrogen_connectivity
+import hydrogen_parametrization
+import modify_gradients
+from scitbx import matrix
+
+#
+def exercise(pdb_str, eps,idealize):
+  mon_lib_srv = monomer_library.server.server()
+  ener_lib = monomer_library.server.ener_lib()
+  processed_pdb_file = monomer_library.pdb_interpretation.process(
+      mon_lib_srv    = mon_lib_srv,
+      ener_lib       = ener_lib,
+      raw_records    = pdb_str,
+      force_symmetry = True)
+  xray_structure = processed_pdb_file.xray_structure()
+  geometry = processed_pdb_file.geometry_restraints_manager(
+      show_energies      = False,
+      plain_pairs_radius = 5.0)
+  sites_cart = xray_structure.sites_cart()
+  es = geometry.energies_sites(
+    sites_cart = sites_cart,
+    compute_gradients = True)
+  g_analytical = es.gradients
+#
+  pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy
+  restraints_manager = mmtbx.restraints.manager(
+    geometry      = geometry,
+    normalization = False)
+  bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
+    sites_cart = xray_structure.sites_cart())
+  angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
+  dihedral_proxies = restraints_manager.geometry.dihedral_proxies
+  names = list(pdb_hierarchy.atoms().extract_name())
+  atoms_list = list(pdb_hierarchy.atoms_with_labels())
+  hd_selection = xray_structure.hd_selection()
+  connectivity = hydrogen_connectivity.determine_H_neighbors(
+    geometry_restraints   = geometry,
+    bond_proxies          = bond_proxies_simple,
+    angle_proxies         = angle_proxies,
+    dihedral_proxies      = dihedral_proxies,
+    hd_selection          = hd_selection,
+    sites_cart            = sites_cart)
+  h_parameterization = hydrogen_parametrization.get_h_parameterization(
+    connectivity   = connectivity,
+    sites_cart     = sites_cart,
+    idealize       = idealize)
+
+  for ih in h_parameterization.keys():
+    h_obj = hydrogen_parametrization.generate_H_positions(
+      sites_cart = sites_cart,
+      ih         = ih,
+      para_info  = h_parameterization[ih])
+    #print 'distance before changing H', (h_obj.rH_gen - matrix.col(sites_cart[ih])).length()
+    sites_cart[ih] = tuple(h_obj.rH_gen)
+    xray_structure.set_sites_cart(sites_cart)
+    h_obj = hydrogen_parametrization.generate_H_positions(
+      sites_cart = sites_cart,
+      ih         = ih,
+      para_info  = h_parameterization[ih])
+    assert ((h_obj.rH_gen - matrix.col(sites_cart[ih])).length() < 0.001), \
+       'H atom was not reconstructed correctly, please check '
+
+
+  #for i in g_analytical:
+  #  print i
+  #print '----------'
+  g_analytical = geometry.energies_sites(
+    sites_cart = sites_cart,
+    compute_gradients = True).gradients
+  modify_gradients.modify_gradients(
+    sites_cart          = sites_cart,
+    h_parameterization  = h_parameterization,
+    grads               = g_analytical)
+
+  #
+  ex = [eps,0,0]
+  ey = [0,eps,0]
+  ez = [0,0,eps]
+  g_fd = flex.vec3_double()
+  for i_site in xrange(sites_cart.size()):
+    g_fd_i = []
+    for e in [ex,ey,ez]:
+      ts = []
+      for sign in [-1,1]:
+        sites_cart_ = sites_cart.deep_copy()
+        sites_cart_[i_site] = [
+          sites_cart_[i_site][j]+e[j]*sign for j in xrange(3)]
+        # after shift, recalculate H position
+        for ih in h_parameterization.keys():
+          h_obj = hydrogen_parametrization.generate_H_positions(
+            sites_cart = sites_cart_,
+            ih         = ih,
+            para_info  = h_parameterization[ih])
+          sites_cart_[ih] = tuple(h_obj.rH_gen)
+        ts.append(geometry.energies_sites(
+          sites_cart = sites_cart_,
+          compute_gradients = False).target)
+      g_fd_i.append((ts[1]-ts[0])/(2*eps))
+    g_fd.append(g_fd_i)
+
+  for g1, g2 in zip(g_analytical, g_fd):
+    print g1,g2
+    assert approx_equal(g1,g2, 1.e-4)
+  print '*'*79
+
+def print_para(h_parameterization, ih):
+  hp = h_parameterization[ih]
+  print hp.htype, 'ih', ih, ', a0 ', hp.a0, ', a1 ', hp.a1, 'a2 = ', hp.a2, \
+    'a = ', hp.a, 'b = ', hp.b, 'h = ', hp.h, 'dist_h =', hp.dist_h
+
+
+#Alanine
+pdb_str_01 = """
+CRYST1   11.863   12.347   12.550  90.00  90.00  90.00 P 1
+SCALE1      0.084296  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.080991  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.079681        0.00000
+ATOM      1  N   ALA B  84       7.017   5.016   7.287  1.00 10.00           N
+ATOM      2  CA  ALA B  84       5.784   5.706   7.445  1.00 10.00           C
+ATOM      3  C   ALA B  84       5.129   5.694   6.011  1.00 10.00           C
+ATOM      4  O   ALA B  84       5.714   5.812   4.900  1.00 10.00           O
+ATOM      5  CB  ALA B  84       6.261   7.426   7.444  1.00 10.00           C
+ATOM      6  HA  ALA B  84       5.269   5.787   7.982  1.00 10.00           H
+ATOM      7  HB1 ALA B  84       5.337   8.016   7.304  1.00 10.00           H
+ATOM      8  HB2 ALA B  84       6.549   7.313   8.459  1.00 10.00           H
+ATOM      9  HB3 ALA B  84       6.616   7.667   6.795  1.00 10.00           H
+"""
+# Arginine:
+pdb_str_02 = """
+CRYST1   14.446   16.451   11.913  90.00  90.00  90.00 P 1
+SCALE1      0.069223  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.060787  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.083942        0.00000
+ATOM      1  N   ARG A  50       6.642  11.452   6.691  1.00 10.00           N
+ATOM      2  CA  ARG A  50       7.570  10.240   6.682  1.00 10.00           C
+ATOM      3  C   ARG A  50       9.005  10.280   6.707  1.00 10.00           C
+ATOM      4  O   ARG A  50       9.602  11.108   6.810  1.00 10.00           O
+ATOM      5  CB  ARG A  50       6.962   9.371   5.075  1.00 10.00           C
+ATOM      6  CG  ARG A  50       5.382   9.275   4.806  1.00 10.00           C
+ATOM      7  CD  ARG A  50       4.830   7.928   6.084  1.00 10.00           C
+ATOM      8  NE  ARG A  50       5.434   6.841   5.374  1.00 10.00           N
+ATOM      9  CZ  ARG A  50       6.450   6.116   6.035  1.00 10.00           C
+ATOM     10  NH1 ARG A  50       7.392   6.694   6.803  1.00 10.00           N
+ATOM     11  NH2 ARG A  50       6.898   5.026   5.305  1.00 10.00           N
+ATOM     12  HA  ARG A  50       6.817   9.405   7.203  1.00 10.00           H
+ATOM     13  HB2 ARG A  50       7.451   9.961   4.409  1.00 10.00           H
+ATOM     14  HB3 ARG A  50       7.391   8.588   5.297  1.00 10.00           H
+ATOM     15  HG2 ARG A  50       5.016  10.089   5.174  1.00 10.00           H
+ATOM     16  HG3 ARG A  50       5.385   9.031   4.141  1.00 10.00           H
+ATOM     17  HD2 ARG A  50       5.268   8.147   6.643  1.00 10.00           H
+ATOM     18  HD3 ARG A  50       3.909   8.226   5.899  1.00 10.00           H
+ATOM     19  HE  ARG A  50       5.208   6.478   4.962  1.00 10.00           H
+ATOM     20 HH11 ARG A  50       6.970   7.301   7.302  1.00 10.00           H
+ATOM     21 HH12 ARG A  50       7.866   6.356   7.069  1.00 10.00           H
+ATOM     22 HH21 ARG A  50       6.604   4.626   5.005  1.00 10.00           H
+ATOM     23 HH22 ARG A  50       7.522   4.396   5.734  1.00 10.00           H
+"""
+#Asparagine
+pdb_str_03 = """
+CRYST1   13.889   13.738   13.126  90.00  90.00  90.00 P 1
+SCALE1      0.071999  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.072791  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.076185        0.00000
+ATOM      1  N   ASN A  35       5.108   6.598   6.668  1.00 10.00           N
+ATOM      2  CA  ASN A  35       6.071   7.634   7.159  1.00 10.00           C
+ATOM      3  C   ASN A  35       6.113   8.702   6.163  1.00 10.00           C
+ATOM      4  O   ASN A  35       5.556   8.558   4.817  1.00 10.00           O
+ATOM      5  CB  ASN A  35       7.439   6.520   6.538  1.00 10.00           C
+ATOM      6  CG  ASN A  35       7.597   5.465   7.487  1.00 10.00           C
+ATOM      7  OD1 ASN A  35       6.743   4.980   8.032  1.00 10.00           O
+ATOM      8  ND2 ASN A  35       8.841   4.827   7.318  1.00 10.00           N
+ATOM      9  HA  ASN A  35       6.387   7.612   7.973  1.00 10.00           H
+ATOM     10  HB2 ASN A  35       7.452   6.464   5.548  1.00 10.00           H
+ATOM     11  HB3 ASN A  35       8.153   7.444   6.485  1.00 10.00           H
+ATOM     12 HD21 ASN A  35       9.022   4.464   7.804  1.00 10.00           H
+ATOM     13 HD22 ASN A  35       9.521   5.284   7.192  1.00 10.00           H
+"""
+# Aspartic Acid
+pdb_str_04 = """
+CRYST1   14.178   13.514   11.978  90.00  90.00  90.00 P 1
+SCALE1      0.070532  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.073997  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.083486        0.00000
+ATOM      1  N   ASP A 121       4.844   6.380   4.967  1.00 10.00           N
+ATOM      2  CA  ASP A 121       6.237   6.469   5.750  1.00 10.00           C
+ATOM      3  C   ASP A 121       5.959   7.525   7.138  1.00 10.00           C
+ATOM      4  O   ASP A 121       5.683   8.452   6.797  1.00 10.00           O
+ATOM      5  CB  ASP A 121       7.582   6.234   5.277  1.00 10.00           C
+ATOM      6  CG  ASP A 121       8.550   6.163   6.136  1.00 10.00           C
+ATOM      7  OD1 ASP A 121       9.035   5.174   6.337  1.00 10.00           O
+ATOM      8  OD2 ASP A 121       9.115   7.253   6.644  1.00 10.00           O
+ATOM      9  HA  ASP A 121       6.173   5.442   6.266  1.00 10.00           H
+ATOM     10  HB2 ASP A 121       7.509   5.748   4.630  1.00 10.00           H
+ATOM     11  HB3 ASP A 121       7.714   7.340   4.588  1.00 10.00           H
+"""
+# Cysteine
+pdb_str_05 = """
+CRYST1   12.081   11.399   14.639  90.00  90.00  90.00 P 1
+SCALE1      0.082775  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.087727  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.068311        0.00000
+ATOM      1  N   CYS A  96       4.823   5.221   7.409  1.00 10.00           N
+ATOM      2  CA  CYS A  96       6.273   5.028   7.456  1.00 10.00           C
+ATOM      3  C   CYS A  96       6.932   5.026   9.008  1.00 10.00           C
+ATOM      4  O   CYS A  96       7.163   5.942   9.454  1.00 10.00           O
+ATOM      5  CB  CYS A  96       6.905   6.223   6.813  1.00 10.00           C
+ATOM      6  SG  CYS A  96       6.265   6.407   5.142  1.00 10.00           S
+ATOM      7  HA  CYS A  96       6.714   4.104   6.960  1.00 10.00           H
+ATOM      8  HB2 CYS A  96       6.849   7.034   6.999  1.00 10.00           H
+ATOM      9  HB3 CYS A  96       8.130   6.106   6.575  1.00 10.00           H
+ATOM     10  HG  CYS A  96       6.882   7.238   4.630  1.00 10.00           H
+"""
+# Glutamine
+pdb_str_06 = """
+CRYST1   14.547   13.375   15.374  90.00  90.00  90.00 P 1
+SCALE1      0.068743  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.074766  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.065045        0.00000
+ATOM      1  N   GLN A  70       6.344   7.129   5.035  1.00 10.00           N
+ATOM      2  CA  GLN A  70       6.644   6.208   6.336  1.00 10.00           C
+ATOM      3  C   GLN A  70       5.352   6.178   6.970  1.00 10.00           C
+ATOM      4  O   GLN A  70       5.126   4.898   7.373  1.00 10.00           O
+ATOM      5  CB  GLN A  70       7.877   6.998   7.230  1.00 10.00           C
+ATOM      6  CG  GLN A  70       7.974   6.681   8.575  1.00 10.00           C
+ATOM      7  CD  GLN A  70       9.249   7.237   9.063  1.00 10.00           C
+ATOM      8  OE1 GLN A  70       9.559   8.209   8.935  1.00 10.00           O
+ATOM      9  NE2 GLN A  70       9.506   6.762  10.402  1.00 10.00           N
+ATOM     10  HA  GLN A  70       7.249   5.584   6.195  1.00 10.00           H
+ATOM     11  HB2 GLN A  70       8.663   7.126   6.476  1.00 10.00           H
+ATOM     12  HB3 GLN A  70       7.331   8.179   7.116  1.00 10.00           H
+ATOM     13  HG2 GLN A  70       7.234   6.385   8.986  1.00 10.00           H
+ATOM     14  HG3 GLN A  70       8.549   5.809   8.279  1.00 10.00           H
+ATOM     15 HE21 GLN A  70       9.114   6.119  10.657  1.00 10.00           H
+ATOM     16 HE22 GLN A  70      10.235   7.102  10.671  1.00 10.00           H
+"""
+# Glutamic Acid
+pdb_str_07 = """
+CRYST1   15.127   14.767   13.161  90.00  90.00  90.00 P 1
+SCALE1      0.066107  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.067719  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.075982        0.00000
+ATOM      1  N   GLU A  24       9.022   8.042   4.852  1.00 10.00           N
+ATOM      2  CA  GLU A  24       8.611   8.384   6.596  1.00 10.00           C
+ATOM      3  C   GLU A  24       9.851   8.805   7.028  1.00 10.00           C
+ATOM      4  O   GLU A  24      10.208   9.672   7.827  1.00 10.00           O
+ATOM      5  CB  GLU A  24       8.163   7.285   7.084  1.00 10.00           C
+ATOM      6  CG  GLU A  24       6.610   6.769   6.720  1.00 10.00           C
+ATOM      7  CD  GLU A  24       6.069   5.633   7.216  1.00 10.00           C
+ATOM      8  OE1 GLU A  24       5.180   5.275   7.007  1.00 10.00           O
+ATOM      9  OE2 GLU A  24       6.826   5.071   8.182  1.00 10.00           O
+ATOM     10  HA  GLU A  24       8.239   8.942   6.618  1.00 10.00           H
+ATOM     11  HB2 GLU A  24       8.834   6.577   7.107  1.00 10.00           H
+ATOM     12  HB3 GLU A  24       8.074   7.246   7.880  1.00 10.00           H
+ATOM     13  HG2 GLU A  24       5.925   7.518   6.600  1.00 10.00           H
+ATOM     14  HG3 GLU A  24       6.944   6.683   5.520  1.00 10.00           H
+"""
+# Glycine
+pdb_str_08 = """
+CRYST1   13.147   11.167   10.709  90.00  90.00  90.00 P 1
+SCALE1      0.076063  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.089550  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.093379        0.00000
+ATOM      1  N   GLY B  93       8.304   6.036   5.544  1.00 10.00           N
+ATOM      2  CA  GLY B  93       7.319   4.905   5.595  1.00 10.00           C
+ATOM      3  C   GLY B  93       5.882   5.364   4.847  1.00 10.00           C
+ATOM      4  O   GLY B  93       5.143   5.776   5.738  1.00 10.00           O
+ATOM      5  HA2 GLY B  93       7.870   4.339   5.372  1.00 10.00           H
+ATOM      6  HA3 GLY B  93       7.123   4.618   6.529  1.00 10.00           H
+"""
+
+# Histidine
+pdb_str_09 = """
+CRYST1   12.286   14.857   15.512  90.00  90.00  90.00 P 1
+SCALE1      0.081393  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.067308  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.064466        0.00000
+ATOM      1  N   HIS A  34       7.292   7.714   9.065  1.00 10.00           N
+ATOM      2  CA  HIS A  34       6.363   8.592   8.255  1.00 10.00           C
+ATOM      3  C   HIS A  34       6.024   9.792   9.401  1.00 10.00           C
+ATOM      4  O   HIS A  34       6.246   9.507  10.701  1.00 10.00           O
+ATOM      5  CB  HIS A  34       4.951   8.082   8.105  1.00 10.00           C
+ATOM      6  CG  HIS A  34       5.323   6.839   7.073  1.00 10.00           C
+ATOM      7  ND1 HIS A  34       5.021   7.108   5.763  1.00 10.00           N
+ATOM      8  CD2 HIS A  34       5.317   5.623   7.295  1.00 10.00           C
+ATOM      9  CE1 HIS A  34       5.333   5.771   5.081  1.00 10.00           C
+ATOM     10  NE2 HIS A  34       5.601   4.927   6.077  1.00 10.00           N
+ATOM     11  HA  HIS A  34       6.539   9.000   7.667  1.00 10.00           H
+ATOM     12  HB2 HIS A  34       4.722   7.760   8.940  1.00 10.00           H
+ATOM     13  HB3 HIS A  34       4.532   8.544   7.770  1.00 10.00           H
+ATOM     14  HD2 HIS A  34       5.433   5.063   7.987  1.00 10.00           H
+ATOM     15  HE1 HIS A  34       5.162   5.891   3.986  1.00 10.00           H
+"""
+# Isoleucine
+pdb_str_10 = """
+CRYST1   14.040   13.236   13.685  90.00  90.00  90.00 P 1
+SCALE1      0.071225  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075552  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.073073        0.00000
+ATOM      1  N   ILE B  81       6.774   6.654   8.528  1.00 10.00           N
+ATOM      2  CA  ILE B  81       6.248   6.915   7.265  1.00 10.00           C
+ATOM      3  C   ILE B  81       5.064   5.915   6.912  1.00 10.00           C
+ATOM      4  O   ILE B  81       4.891   4.882   7.591  1.00 10.00           O
+ATOM      5  CB  ILE B  81       7.374   7.060   6.201  1.00 10.00           C
+ATOM      6  CG1 ILE B  81       7.928   5.644   6.013  1.00 10.00           C
+ATOM      7  CG2 ILE B  81       8.450   8.146   6.766  1.00 10.00           C
+ATOM      8  CD1 ILE B  81       9.014   5.797   4.814  1.00 10.00           C
+ATOM      9  HA  ILE B  81       5.805   7.933   7.605  1.00 10.00           H
+ATOM     10  HB  ILE B  81       6.968   7.172   5.320  1.00 10.00           H
+ATOM     11 HG12 ILE B  81       8.488   5.662   7.117  1.00 10.00           H
+ATOM     12 HG13 ILE B  81       7.287   4.921   5.901  1.00 10.00           H
+ATOM     13 HG21 ILE B  81       8.841   8.468   5.740  1.00 10.00           H
+ATOM     14 HG22 ILE B  81       7.916   9.169   6.877  1.00 10.00           H
+ATOM     15 HG23 ILE B  81       8.650   8.115   7.619  1.00 10.00           H
+ATOM     16 HD11 ILE B  81       9.263   4.839   4.994  1.00 10.00           H
+ATOM     17 HD12 ILE B  81       8.679   5.953   4.216  1.00 10.00           H
+ATOM     18 HD13 ILE B  81       9.551   6.406   5.137  1.00 10.00           H
+"""
+
+# Leucine
+pdb_str_11 = """
+CRYST1   13.550   15.138   12.702  90.00  90.00  90.00 P 1
+SCALE1      0.073801  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.066059  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.078728        0.00000
+ATOM      1  N   LEU B  94       6.273   6.593   4.935  1.00 10.00           N
+ATOM      2  CA  LEU B  94       7.641   7.224   5.628  1.00 10.00           C
+ATOM      3  C   LEU B  94       8.422   6.084   6.113  1.00 10.00           C
+ATOM      4  O   LEU B  94       8.638   5.157   5.672  1.00 10.00           O
+ATOM      5  CB  LEU B  94       7.066   8.117   6.861  1.00 10.00           C
+ATOM      6  CG  LEU B  94       6.206   9.526   6.450  1.00 10.00           C
+ATOM      7  CD1 LEU B  94       5.918  10.237   7.727  1.00 10.00           C
+ATOM      8  CD2 LEU B  94       4.891   9.251   5.577  1.00 10.00           C
+ATOM      9  HA  LEU B  94       7.928   7.858   4.837  1.00 10.00           H
+ATOM     10  HB2 LEU B  94       6.600   7.347   7.185  1.00 10.00           H
+ATOM     11  HB3 LEU B  94       7.833   8.235   7.219  1.00 10.00           H
+ATOM     12  HG  LEU B  94       7.048   9.715   5.954  1.00 10.00           H
+ATOM     13 HD11 LEU B  94       5.564  11.132   7.460  1.00 10.00           H
+ATOM     14 HD12 LEU B  94       5.689   9.764   8.196  1.00 10.00           H
+ATOM     15 HD13 LEU B  94       6.901  10.482   8.316  1.00 10.00           H
+ATOM     16 HD21 LEU B  94       4.622   9.894   5.701  1.00 10.00           H
+ATOM     17 HD22 LEU B  94       5.223   8.782   4.700  1.00 10.00           H
+ATOM     18 HD23 LEU B  94       4.327   8.453   6.346  1.00 10.00           H
+"""
+
+# Lysine
+pdb_str_12 = """
+CRYST1   14.470   17.497   11.416  90.00  90.00  90.00 P 1
+SCALE1      0.069109  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.057153  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.087596        0.00000
+ATOM      1  N   LYS A  76       4.981  10.159   4.860  1.00 10.00           N
+ATOM      2  CA  LYS A  76       5.876  10.071   6.050  1.00 10.00           C
+ATOM      3  C   LYS A  76       6.304  11.829   6.325  1.00 10.00           C
+ATOM      4  O   LYS A  76       5.161  12.466   6.576  1.00 10.00           O
+ATOM      5  CB  LYS A  76       7.337   9.388   5.906  1.00 10.00           C
+ATOM      6  CG  LYS A  76       7.118   7.831   5.788  1.00 10.00           C
+ATOM      7  CD  LYS A  76       8.453   7.428   5.736  1.00 10.00           C
+ATOM      8  CE  LYS A  76       8.134   5.835   5.433  1.00 10.00           C
+ATOM      9  NZ  LYS A  76       9.426   4.919   5.392  1.00 10.00           N
+ATOM     10  HA  LYS A  76       5.397  10.002   6.803  1.00 10.00           H
+ATOM     11  HB2 LYS A  76       7.655   9.920   5.112  1.00 10.00           H
+ATOM     12  HB3 LYS A  76       7.624   9.427   6.543  1.00 10.00           H
+ATOM     13  HG2 LYS A  76       6.592   7.548   6.676  1.00 10.00           H
+ATOM     14  HG3 LYS A  76       6.469   7.536   5.090  1.00 10.00           H
+ATOM     15  HD2 LYS A  76       8.768   7.405   4.630  1.00 10.00           H
+ATOM     16  HD3 LYS A  76       9.025   7.220   6.546  1.00 10.00           H
+ATOM     17  HE2 LYS A  76       7.580   5.564   6.304  1.00 10.00           H
+ATOM     18  HE3 LYS A  76       7.440   5.632   4.831  1.00 10.00           H
+ATOM     19  HZ1 LYS A  76       9.379   4.186   5.442  1.00 10.00           H
+ATOM     20  HZ2 LYS A  76       9.725   5.254   4.549  1.00 10.00           H
+ATOM     21  HZ3 LYS A  76       9.965   5.006   6.093  1.00 10.00           H
+"""
+
+# Methionine
+pdb_str_13 = """
+CRYST1   13.775   12.351   15.645  90.00  90.00  90.00 P 1
+SCALE1      0.072595  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.080965  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.063918        0.00000
+ATOM      1  N   MET B  37       7.603   5.375   6.446  1.00 10.00           N
+ATOM      2  CA  MET B  37       6.723   6.349   6.607  1.00 10.00           C
+ATOM      3  C   MET B  37       6.323   7.146   5.241  1.00 10.00           C
+ATOM      4  O   MET B  37       5.062   7.131   4.970  1.00 10.00           O
+ATOM      5  CB  MET B  37       6.932   7.378   7.483  1.00 10.00           C
+ATOM      6  CG  MET B  37       7.258   6.900   9.095  1.00 10.00           C
+ATOM      7  SD  MET B  37       8.785   5.488   9.096  1.00 10.00           S
+ATOM      8  CE  MET B  37       8.590   5.149  10.517  1.00 10.00           C
+ATOM      9  HA  MET B  37       5.885   6.031   6.839  1.00 10.00           H
+ATOM     10  HB2 MET B  37       7.862   7.670   7.351  1.00 10.00           H
+ATOM     11  HB3 MET B  37       6.364   7.936   7.854  1.00 10.00           H
+ATOM     12  HG2 MET B  37       7.509   7.571   9.764  1.00 10.00           H
+ATOM     13  HG3 MET B  37       6.614   6.240   9.246  1.00 10.00           H
+ATOM     14  HE1 MET B  37       9.624   4.355  10.601  1.00 10.00           H
+ATOM     15  HE2 MET B  37       9.070   5.811  11.280  1.00 10.00           H
+ATOM     16  HE3 MET B  37       8.003   4.499  11.109  1.00 10.00           H
+"""
+
+# Phenylanaline
+pdb_str_14 = """
+CRYST1   11.802   16.578   13.976  90.00  90.00  90.00 P 1
+SCALE1      0.084731  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.060321  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.071551        0.00000
+ATOM      1  N   PHE A  63       6.412   9.770   7.572  1.00 10.00           N
+ATOM      2  CA  PHE A  63       6.289   9.157   6.198  1.00 10.00           C
+ATOM      3  C   PHE A  63       6.292  10.223   5.110  1.00 10.00           C
+ATOM      4  O   PHE A  63       6.451  11.437   5.241  1.00 10.00           O
+ATOM      5  CB  PHE A  63       5.580   8.319   6.063  1.00 10.00           C
+ATOM      6  CG  PHE A  63       5.743   6.958   6.849  1.00 10.00           C
+ATOM      7  CD1 PHE A  63       6.569   6.117   6.768  1.00 10.00           C
+ATOM      8  CD2 PHE A  63       5.050   7.055   8.045  1.00 10.00           C
+ATOM      9  CE1 PHE A  63       6.802   5.054   7.620  1.00 10.00           C
+ATOM     10  CE2 PHE A  63       5.331   6.153   8.840  1.00 10.00           C
+ATOM     11  CZ  PHE A  63       6.270   5.103   8.840  1.00 10.00           C
+ATOM     12  HA  PHE A  63       7.224   9.004   5.959  1.00 10.00           H
+ATOM     13  HB2 PHE A  63       4.619   8.277   5.936  1.00 10.00           H
+ATOM     14  HB3 PHE A  63       5.713   7.612   5.105  1.00 10.00           H
+ATOM     15  HD1 PHE A  63       7.301   6.004   5.919  1.00 10.00           H
+ATOM     16  HD2 PHE A  63       4.436   7.641   8.314  1.00 10.00           H
+ATOM     17  HE1 PHE A  63       7.224   4.307   7.397  1.00 10.00           H
+ATOM     18  HE2 PHE A  63       4.727   6.172   9.862  1.00 10.00           H
+ATOM     19  HZ  PHE A  63       6.045   4.444   9.418  1.00 10.00           H
+"""
+
+# Proline
+pdb_str_15 = """
+CRYST1   12.293   14.006   12.486  90.00  90.00  90.00 P 1
+SCALE1      0.081347  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.071398  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.080090        0.00000
+ATOM      1  N   PRO A   4       4.920   7.325   6.411  1.00 10.00           N
+ATOM      2  CA  PRO A   4       6.162   6.834   5.666  1.00 10.00           C
+ATOM      3  C   PRO A   4       6.405   5.341   5.894  1.00 10.00           C
+ATOM      4  O   PRO A   4       7.098   4.853   4.957  1.00 10.00           O
+ATOM      5  CB  PRO A   4       7.380   7.894   6.618  1.00 10.00           C
+ATOM      6  CG  PRO A   4       6.640   8.868   7.204  1.00 10.00           C
+ATOM      7  CD  PRO A   4       5.415   8.476   7.560  1.00 10.00           C
+ATOM      8  HA  PRO A   4       6.250   7.067   4.776  1.00 10.00           H
+ATOM      9  HB2 PRO A   4       7.921   7.337   7.346  1.00 10.00           H
+ATOM     10  HB3 PRO A   4       8.016   8.080   5.830  1.00 10.00           H
+ATOM     11  HG2 PRO A   4       7.167   9.579   7.723  1.00 10.00           H
+ATOM     12  HG3 PRO A   4       6.492   9.649   6.528  1.00 10.00           H
+ATOM     13  HD2 PRO A   4       5.306   8.082   8.255  1.00 10.00           H
+ATOM     14  HD3 PRO A   4       4.496   9.094   7.222  1.00 10.00           H
+"""
+
+# Serine
+pdb_str_16 = """
+CRYST1   12.893   12.708   12.721  90.00  90.00  90.00 P 1
+SCALE1      0.077561  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.078691  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.078610        0.00000
+ATOM      1  N   SER A  73       5.347   5.750   5.075  1.00 10.00           N
+ATOM      2  CA  SER A  73       5.894   5.594   6.421  1.00 10.00           C
+ATOM      3  C   SER A  73       5.946   7.282   6.996  1.00 10.00           C
+ATOM      4  O   SER A  73       5.026   7.627   7.452  1.00 10.00           O
+ATOM      5  CB  SER A  73       7.501   5.019   6.235  1.00 10.00           C
+ATOM      6  OG  SER A  73       8.063   4.929   7.844  1.00 10.00           O
+ATOM      7  HA  SER A  73       5.300   5.295   7.021  1.00 10.00           H
+ATOM      8  HB2 SER A  73       7.399   4.067   6.115  1.00 10.00           H
+ATOM      9  HB3 SER A  73       8.072   5.776   5.734  1.00 10.00           H
+ATOM     10  HG  SER A  73       8.499   4.647   7.768  1.00 10.00           H
+"""
+
+# Threonine
+pdb_str_17 = """
+CRYST1   11.909   12.199   14.459  90.00  90.00  90.00 P 1
+SCALE1      0.083970  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.081974  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.069161        0.00000
+ATOM      1  N   THR A  68       6.967   7.035   7.598  1.00 10.00           N
+ATOM      2  CA  THR A  68       5.580   6.835   7.409  1.00 10.00           C
+ATOM      3  C   THR A  68       4.981   6.180   8.674  1.00 10.00           C
+ATOM      4  O   THR A  68       6.079   5.829   9.469  1.00 10.00           O
+ATOM      5  CB  THR A  68       5.071   6.452   6.355  1.00 10.00           C
+ATOM      6  OG1 THR A  68       5.819   5.087   6.249  1.00 10.00           O
+ATOM      7  CG2 THR A  68       5.178   7.219   5.045  1.00 10.00           C
+ATOM      8  HA  THR A  68       4.978   7.946   7.649  1.00 10.00           H
+ATOM      9  HB  THR A  68       3.853   6.156   6.182  1.00 10.00           H
+ATOM     10  HG1 THR A  68       6.272   5.006   6.437  1.00 10.00           H
+ATOM     11 HG21 THR A  68       5.183   6.722   4.241  1.00 10.00           H
+ATOM     12 HG22 THR A  68       4.943   8.119   5.225  1.00 10.00           H
+ATOM     13 HG23 THR A  68       6.305   7.117   4.778  1.00 10.00           H
+"""
+
+# Triptophane
+pdb_str_18 = """
+CRYST1   12.502   12.982   18.312  90.00  90.00  90.00 P 1
+SCALE1      0.079987  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.077030  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.054609        0.00000
+ATOM      1  N   TRP A  49       7.019   6.036   6.855  1.00 10.00           N
+ATOM      2  CA  TRP A  49       6.009   5.654   7.261  1.00 10.00           C
+ATOM      3  C   TRP A  49       5.132   5.145   5.877  1.00 10.00           C
+ATOM      4  O   TRP A  49       5.267   5.983   5.027  1.00 10.00           O
+ATOM      5  CB  TRP A  49       5.058   6.563   8.316  1.00 10.00           C
+ATOM      6  CG  TRP A  49       6.065   6.967   9.207  1.00 10.00           C
+ATOM      7  CD1 TRP A  49       6.863   7.994   9.455  1.00 10.00           C
+ATOM      8  CD2 TRP A  49       5.959   6.505  10.743  1.00 10.00           C
+ATOM      9  NE1 TRP A  49       7.435   8.004  10.831  1.00 10.00           N
+ATOM     10  CE2 TRP A  49       6.803   7.200  11.335  1.00 10.00           C
+ATOM     11  CE3 TRP A  49       5.119   5.187  11.318  1.00 10.00           C
+ATOM     12  CZ2 TRP A  49       7.129   6.620  12.976  1.00 10.00           C
+ATOM     13  CZ3 TRP A  49       5.299   5.155  12.457  1.00 10.00           C
+ATOM     14  CH2 TRP A  49       6.374   5.658  13.211  1.00 10.00           C
+ATOM     15  HA  TRP A  49       6.139   4.842   7.608  1.00 10.00           H
+ATOM     16  HB2 TRP A  49       5.070   7.319   7.481  1.00 10.00           H
+ATOM     17  HB3 TRP A  49       4.305   6.336   8.251  1.00 10.00           H
+ATOM     18  HD1 TRP A  49       7.226   8.582   8.796  1.00 10.00           H
+ATOM     19  HE1 TRP A  49       8.120   8.555  10.929  1.00 10.00           H
+ATOM     20  HE3 TRP A  49       4.311   5.074  10.564  1.00 10.00           H
+ATOM     21  HZ2 TRP A  49       7.806   7.092  13.214  1.00 10.00           H
+ATOM     22  HZ3 TRP A  49       4.914   4.129  12.933  1.00 10.00           H
+ATOM     23  HH2 TRP A  49       6.353   5.513  14.064  1.00 10.00           H
+"""
+
+# Tyrosine
+pdb_str_19 = """
+CRYST1   17.955   13.272   13.095  90.00  90.00  90.00 P 1
+SCALE1      0.055695  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.075347  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.076365        0.00000
+ATOM      1  N   TYR A 139      10.057   7.968   5.049  1.00 10.00           N
+ATOM      2  CA  TYR A 139      10.657   7.531   6.379  1.00 10.00           C
+ATOM      3  C   TYR A 139      12.203   7.725   6.416  1.00 10.00           C
+ATOM      4  O   TYR A 139      12.999   8.272   7.373  1.00 10.00           O
+ATOM      5  CB  TYR A 139      10.644   6.145   6.711  1.00 10.00           C
+ATOM      6  CG  TYR A 139       9.159   5.899   6.690  1.00 10.00           C
+ATOM      7  CD1 TYR A 139       8.503   5.230   5.513  1.00 10.00           C
+ATOM      8  CD2 TYR A 139       8.317   6.046   8.121  1.00 10.00           C
+ATOM      9  CE1 TYR A 139       6.876   4.938   5.643  1.00 10.00           C
+ATOM     10  CE2 TYR A 139       7.209   5.706   8.077  1.00 10.00           C
+ATOM     11  CZ  TYR A 139       6.420   5.365   6.855  1.00 10.00           C
+ATOM     12  OH  TYR A 139       5.027   4.949   7.088  1.00 10.00           O
+ATOM     13  HA  TYR A 139      10.303   8.174   6.785  1.00 10.00           H
+ATOM     14  HB2 TYR A 139      10.989   5.647   5.882  1.00 10.00           H
+ATOM     15  HB3 TYR A 139      10.828   5.883   7.586  1.00 10.00           H
+ATOM     16  HD1 TYR A 139       8.618   5.356   4.741  1.00 10.00           H
+ATOM     17  HD2 TYR A 139       8.841   6.121   8.546  1.00 10.00           H
+ATOM     18  HE1 TYR A 139       6.432   5.030   4.892  1.00 10.00           H
+ATOM     19  HE2 TYR A 139       6.780   5.619   9.066  1.00 10.00           H
+ATOM     20  HH  TYR A 139       4.693   5.141   7.840  1.00 10.00           H
+"""
+
+# Valine
+pdb_str_20 = """
+CRYST1   12.396   13.122   13.130  90.00  90.00  90.00 P 1
+SCALE1      0.080671  0.000000  0.000000        0.00000
+SCALE2      0.000000  0.076208  0.000000        0.00000
+SCALE3      0.000000  0.000000  0.076161        0.00000
+ATOM      1  N   VAL B  78       4.820   5.520   7.634  1.00 10.00           N
+ATOM      2  CA  VAL B  78       6.105   5.869   7.138  1.00 10.00           C
+ATOM      3  C   VAL B  78       6.717   7.176   8.084  1.00 10.00           C
+ATOM      4  O   VAL B  78       5.750   8.012   8.022  1.00 10.00           O
+ATOM      5  CB  VAL B  78       6.029   6.374   5.701  1.00 10.00           C
+ATOM      6  CG1 VAL B  78       5.673   4.961   4.874  1.00 10.00           C
+ATOM      7  CG2 VAL B  78       7.349   6.810   5.233  1.00 10.00           C
+ATOM      8  HA  VAL B  78       7.125   5.144   7.411  1.00 10.00           H
+ATOM      9  HB  VAL B  78       5.404   6.962   5.498  1.00 10.00           H
+ATOM     10 HG11 VAL B  78       5.583   5.210   4.024  1.00 10.00           H
+ATOM     11 HG12 VAL B  78       6.062   4.120   5.126  1.00 10.00           H
+ATOM     12 HG13 VAL B  78       4.907   4.924   5.317  1.00 10.00           H
+ATOM     13 HG21 VAL B  78       7.299   7.038   4.409  1.00 10.00           H
+ATOM     14 HG22 VAL B  78       7.542   7.634   5.823  1.00 10.00           H
+ATOM     15 HG23 VAL B  78       7.889   5.963   5.346  1.00 10.00           H
+"""
+
+pdb_list = [pdb_str_01, pdb_str_02, pdb_str_03, pdb_str_04, pdb_str_05,
+  pdb_str_06, pdb_str_07, pdb_str_08, pdb_str_09, pdb_str_10, pdb_str_11,
+  pdb_str_12, pdb_str_13, pdb_str_14, pdb_str_15, pdb_str_16, pdb_str_17,
+  pdb_str_18, pdb_str_19, pdb_str_20]
+#
+pdb_list_name = ['pdb_str_01', 'pdb_str_02', 'pdb_str_03', 'pdb_str_04', 'pdb_str_05',
+  'pdb_str_06', 'pdb_str_07', 'pdb_str_08', 'pdb_str_09', 'pdb_str_10', 'pdb_str_11',
+  'pdb_str_12', 'pdb_str_13', 'pdb_str_14', 'pdb_str_15', 'pdb_str_16', 'pdb_str_17',
+  'pdb_str_18', 'pdb_str_19', 'pdb_str_20']
+
+#pdb_list = [pdb_str_02]
+#pdb_list_name = ['pdb_str_02']
+
+def run():
+  for idealize in [True, False]:
+    for pdb_str, str_name in zip(pdb_list,pdb_list_name):
+      print 'pdb_string:', str_name, 'idealize =', idealize
+      exercise(pdb_str=pdb_str, eps=1.e-4, idealize=idealize)
+
+
+if (__name__ == "__main__"):
+  t0 = time.time()
+  run()
+  #run(sys.argv[1:])
+  print "Time:", round(time.time()-t0, 2)

Added: trunk/mmtbx/hydrogens/tst_run_fd_3.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_run_fd_3.py	                        (rev 0)
+++ trunk/mmtbx/hydrogens/tst_run_fd_3.py	2016-10-05 23:21:00 UTC (rev 25557)
@@ -0,0 +1,453 @@
+from __future__ import division
+import time
+from cctbx.array_family import flex
+from mmtbx import monomer_library
+import mmtbx.monomer_library.server
+import mmtbx.monomer_library.pdb_interpretation
+from libtbx.test_utils import approx_equal
+import hydrogen_connectivity
+import hydrogen_parametrization
+import modify_gradients
+from scitbx import matrix
+
+def exercise(pdb_str, eps, idealize):
+  mon_lib_srv = monomer_library.server.server()
+  ener_lib = monomer_library.server.ener_lib()
+  processed_pdb_file = monomer_library.pdb_interpretation.process(
+      mon_lib_srv    = mon_lib_srv,
+      ener_lib       = ener_lib,
+      raw_records    = pdb_str,
+      force_symmetry = True)
+  xray_structure = processed_pdb_file.xray_structure()
+  geometry = processed_pdb_file.geometry_restraints_manager(
+      show_energies      = False,
+      plain_pairs_radius = 5.0)
+  sites_cart = xray_structure.sites_cart()
+  es = geometry.energies_sites(
+    sites_cart = sites_cart,
+    compute_gradients = True)
+  g_analytical = es.gradients
+#
+  pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy
+  restraints_manager = mmtbx.restraints.manager(
+    geometry      = geometry,
+    normalization = False)
+  bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
+    sites_cart = xray_structure.sites_cart())
+  angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
+  dihedral_proxies = restraints_manager.geometry.dihedral_proxies
+  names = list(pdb_hierarchy.atoms().extract_name())
+  atoms_list = list(pdb_hierarchy.atoms_with_labels())
+  hd_selection = xray_structure.hd_selection()
+  connectivity = hydrogen_connectivity.determine_H_neighbors(
+    geometry_restraints   = geometry,
+    bond_proxies          = bond_proxies_simple,
+    angle_proxies         = angle_proxies,
+    dihedral_proxies      = dihedral_proxies,
+    hd_selection          = hd_selection,
+    sites_cart            = sites_cart)
+  h_parameterization = hydrogen_parametrization.get_h_parameterization(
+    connectivity   = connectivity,
+    sites_cart     = sites_cart,
+    idealize       = idealize)
+
+  for ih in h_parameterization.keys():
+    h_obj = hydrogen_parametrization.generate_H_positions(
+      sites_cart = sites_cart,
+ ...
 
[truncated message content]

Revision: 25556
          http://sourceforge.net/p/cctbx/code/25556
Author:   dcliebschner
Date:     2016-10-05 22:46:36 +0000 (Wed, 05 Oct 2016)
Log Message:
-----------
new name for test

Modified Paths:
--------------
    trunk/mmtbx/run_tests.py

Modified: trunk/mmtbx/run_tests.py
===================================================================
--- trunk/mmtbx/run_tests.py	2016-10-05 22:43:36 UTC (rev 25555)
+++ trunk/mmtbx/run_tests.py	2016-10-05 22:46:36 UTC (rev 25556)
@@ -103,7 +103,7 @@
   "$D/regression/tst_altloc_chain_break.py",
   "$D/hydrogens/build_hydrogens.py",
   "$D/hydrogens/tst.py",
-  "$D/hydrogens/tst_hydrogen_connectivity.py",
+  "$D/hydrogens/tst_h_connectivity.py",
   "$D/hydrogens/tst_h_parametrization_1.py",
   "$D/hydrogens/tst_h_parametrization_2.py",
   "$D/max_lik/tst_maxlik.py",

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25555
          http://sourceforge.net/p/cctbx/code/25555
Author:   bkpoon
Date:     2016-10-05 22:43:36 +0000 (Wed, 05 Oct 2016)
Log Message:
-----------
Update tests for extract_miller_array_from_file function

Modified Paths:
--------------
    trunk/iotbx/regression/tst_reflection_file_utils.py

Modified: trunk/iotbx/regression/tst_reflection_file_utils.py
===================================================================
--- trunk/iotbx/regression/tst_reflection_file_utils.py	2016-10-05 22:42:31 UTC (rev 25554)
+++ trunk/iotbx/regression/tst_reflection_file_utils.py	2016-10-05 22:43:36 UTC (rev 25555)
@@ -710,7 +710,7 @@
     try:
       rfu.extract_miller_array_from_file(file_name="tmp.mtz",type=tp, log=log)
     except Sorry, e:
-      assert str(e)=="Multiple choices available."
+      assert ("Multiple choices available." in str(e))
       sorry_counts += 1
   #
   for tp in [None, "real"]:
@@ -720,7 +720,7 @@
     mtz_dataset.mtz_object().write("tmp.mtz")
     try: rfu.extract_miller_array_from_file(file_name="tmp.mtz",type=tp,log=log)
     except Sorry, e:
-      assert str(e)=="Multiple choices available."
+      assert ("Multiple choices available." in str(e))
       sorry_counts += 1
   #
   mtz_dataset = a3.as_mtz_dataset(column_root_label="A3")
@@ -729,7 +729,7 @@
   mtz_dataset.mtz_object().write("tmp.mtz")
   try: rfu.extract_miller_array_from_file(file_name="tmp.mtz",log=log)
   except Sorry, e:
-    assert str(e)=="Multiple choices available."
+    assert ("Multiple choices available." in str(e))
     sorry_counts += 1
   #
   mtz_dataset = a4.as_mtz_dataset(column_root_label="A4")

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25554
          http://sourceforge.net/p/cctbx/code/25554
Author:   dcliebschner
Date:     2016-10-05 22:42:31 +0000 (Wed, 05 Oct 2016)
Log Message:
-----------
newest version for parameterization

Modified Paths:
--------------
    trunk/mmtbx/hydrogens/hydrogen_connectivity.py
    trunk/mmtbx/hydrogens/hydrogen_parametrization.py
    trunk/mmtbx/hydrogens/tst_h_parametrization_1.py
    trunk/mmtbx/hydrogens/tst_h_parametrization_2.py

Added Paths:
-----------
    trunk/mmtbx/hydrogens/tst_h_connectivity.py

Modified: trunk/mmtbx/hydrogens/hydrogen_connectivity.py
===================================================================
--- trunk/mmtbx/hydrogens/hydrogen_connectivity.py	2016-10-05 22:01:45 UTC (rev 25553)
+++ trunk/mmtbx/hydrogens/hydrogen_connectivity.py	2016-10-05 22:42:31 UTC (rev 25554)
@@ -2,9 +2,12 @@
 # LIBTBX_SET_DISPATCHER_NAME phenix.hydrogen_connectivity
 import sys
 import time
+import math
 import mmtbx.monomer_library.server
 import mmtbx.monomer_library.pdb_interpretation
+from mmtbx.monomer_library.pdb_interpretation import grand_master_phil_str
 import mmtbx.utils
+import iotbx.phil
 from mmtbx import monomer_library
 from cctbx import geometry_restraints
 from scitbx.array_family import flex
@@ -13,6 +16,7 @@
 from libtbx.utils import null_out
 from libtbx.utils import multi_out
 from scitbx import matrix
+from scitbx.math import dihedral_angle
 
 legend = """\
 
@@ -54,6 +58,9 @@
     dist_ideal      = None,
     angle           = None,
     angle_ideal     = None,
+    dihedral_ideal  = None,
+    dihedral        = None,
+    dihe_list       = None,
     reduced_neighbs = None,
     count_H         = None):
     adopt_init_args(self, locals())
@@ -67,15 +74,14 @@
 # [b1]     --> if only one 1-3 neighbor, list also the 1-4 neighbors
 # a0, a1 are objects "atom_info"
 # ----------------------------------------------------------------------------
-def determine_H_neighbors(
-  geometry_restraints, bond_proxies, angle_proxies, hd_selection, sites_cart):
+def determine_H_neighbors(geometry_restraints, bond_proxies, angle_proxies,
+  dihedral_proxies, hd_selection, sites_cart):
   fsc2=geometry_restraints.shell_sym_tables[2].full_simple_connectivity()
   fsc1=geometry_restraints.shell_sym_tables[1].full_simple_connectivity()
   #fsc0=geometry_restraints.shell_sym_tables[0].full_simple_connectivity()
   #hd_selection = xray_structure.hd_selection()
   #sites_cart = xray_structure.sites_cart()
   # Maybe there is better way to get number of atoms?
-  # (needed to select angle proxy)
   n_atoms = len(sites_cart)
   connectivity = {}
   # loop through bond proxies to find H atom and parent atom
@@ -115,14 +121,13 @@
           angle_ideal = angle)
         connectivity[ih][1].append(neighbor)
     reduced_neighbs = []
-    # list of reduced neigbors is needed for finding parameterization
     for atom in connectivity[ih][1]:
       if (not hd_selection[atom.iseq]):
         reduced_neighbs.append(atom)
     (connectivity[ih][0]).reduced_neighbs = reduced_neighbs
     (connectivity[ih][0]).count_H = \
       len(connectivity[ih][1]) - len(reduced_neighbs)
-    # add third neighbors, if necessary
+    # find third neighbors, if necessary
     if (len(reduced_neighbs) == 1):
       connectivity[ih].append([])
       i_second = (reduced_neighbs[0]).iseq
@@ -136,8 +141,24 @@
           if ap:
             neighbor = atom_info(
               iseq = i_third)
+            iselection_dihe = flex.size_t([ih,i_parent,i_second,i_third])
+            dp = dihedral_proxies.proxy_select(
+              n_seq      = n_atoms,
+              iselection = iselection_dihe)
+            if dp:
+              dihedral_id = dp[0].angle_ideal
+              dihedral = dihedral_angle(
+                sites=[sites_cart[i_third], sites_cart[i_second],
+                sites_cart[i_parent],sites_cart[ih]])
+              (connectivity[ih][0]).dihedral = dihedral
+              (connectivity[ih][0]).dihe_list = [ih,i_parent,i_second,i_third]
+              sites_cart_dihe = sites_cart.select(iselection_dihe).deep_copy()
+              delta = dp.deltas(sites_cart=sites_cart_dihe)[0]
+              dihedral_ideal = math.degrees(dihedral) + delta
+              (connectivity[ih][0]).dihedral_ideal = dihedral_ideal
+              #print dihedral_id, delta, math.degrees(dihedral), dihedral_ideal
             connectivity[ih][2].append(neighbor)
-    if (len(reduced_neighbs) == 2 or len(reduced_neighbs) == 3):
+    if (len(reduced_neighbs) == 2):
       ix = i_parent
       iy = (reduced_neighbs[0]).iseq
       iz = (reduced_neighbs[1]).iseq
@@ -145,6 +166,21 @@
       (connectivity[ih][0]).angle_ideal =  angle_proxies.proxy_select(
         n_seq      = n_atoms,
         iselection = iselection)[0].angle_ideal
+    if (len(reduced_neighbs) == 3):
+    # for tetrahedral, all 3 ideal angles are needed
+      angles = []
+      ix = i_parent
+      _list = [(0,1),(1,2),(2,0)]
+      for _i,_j in _list:
+        iy = (reduced_neighbs[_i]).iseq
+        iz = (reduced_neighbs[_j]).iseq
+        iselection = flex.size_t([ix,iy,iz])
+        ap = angle_proxies.proxy_select(
+          n_seq      = n_atoms,
+          iselection = iselection)
+        if ap:
+          angles.append(ap[0].angle_ideal)
+        (connectivity[ih][0]).angle_ideal = angles
   return connectivity
 
 def run(args, out=sys.stdout):
@@ -159,6 +195,14 @@
   print >> log, "phenix.hydrogen_connectivity is running..."
   print >> log, "input parameters:\n", args
 
+# -------------------------------------------------
+#          code to switch off CDL or not
+# -------------------------------------------------
+  params_line = grand_master_phil_str
+  params = iotbx.phil.parse(
+      input_string=params_line, process_includes=True).extract()
+  params.pdb_interpretation.restraints_library.cdl=False
+#
   processed_args = mmtbx.utils.process_command_line_args(
     args=args, log=null_out())
   pdb_filename = processed_args.pdb_file_names[0]
@@ -170,21 +214,20 @@
       ener_lib       = ener_lib,
       file_name      = pdb_filename,
       force_symmetry = True)
-
   pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy
   xray_structure = processed_pdb_file.xray_structure()
   if pdb_filename is not None:
     pdb_str = pdb_hierarchy.as_pdb_string()
 
-  geometry_restraints = processed_pdb_file.geometry_restraints_manager(
+  grm = processed_pdb_file.geometry_restraints_manager(
     show_energies = False)
   restraints_manager = mmtbx.restraints.manager(
-    geometry      = geometry_restraints,
+    geometry      = grm,
     normalization = False)
-
   bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
     sites_cart = xray_structure.sites_cart())
   angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
+  dihedral_proxies = restraints_manager.geometry.dihedral_proxies
 
   names = list(pdb_hierarchy.atoms().extract_name())
   sites_cart = xray_structure.sites_cart()
@@ -194,10 +237,12 @@
 
   print >>log, '\nNow determining connectivity table for H atoms...'
   connectivity = determine_H_neighbors(
-    geometry_restraints   = geometry_restraints,
+    geometry_restraints   = grm,
     bond_proxies          = bond_proxies_simple,
     angle_proxies         = angle_proxies,
-    xray_structure        = xray_structure)
+    dihedral_proxies      = dihedral_proxies,
+    hd_selection          = hd_selection,
+    sites_cart            = sites_cart)
 
   if(0):
     print >>log, '\nHydrogen atom connectivity list'

Modified: trunk/mmtbx/hydrogens/hydrogen_parametrization.py
===================================================================
--- trunk/mmtbx/hydrogens/hydrogen_parametrization.py	2016-10-05 22:01:45 UTC (rev 25553)
+++ trunk/mmtbx/hydrogens/hydrogen_parametrization.py	2016-10-05 22:42:31 UTC (rev 25554)
@@ -7,7 +7,6 @@
 import mmtbx.utils
 from mmtbx import monomer_library
 import iotbx.phil
-#from mmtbx import hydrogens
 from cctbx import geometry_restraints
 from scitbx import matrix
 #from libtbx.utils import Sorry
@@ -46,23 +45,143 @@
 class parameterization_info(object):
   def __init__(
     self,
-    htype      = None,  # type of bond
+    htype      = None,  # type of hydrogen environment
     a0         = None,  # parent atom index
     a1         = None,  # 1-3 neighbor index
     a2         = None,  # 1-3 or 1-4 neighbor index
+    a3         = None,  # 1-3 or 1-4 neighbor index
     a          = None,  # coefficient for reconstruction
     b          = None,  # coefficient for reconstruction
     h          = None,  # coefficient for reconstruction
     phi        = None,  # angle
     alpha      = None,  # angle
+    n          = None,  # parameter for propeller and H2 groups
     dist_h     = None): # measured or ideal distance
     adopt_init_args(self, locals())
 
+# this function catches three cases:
+# 1. planar geometry
+# 2. two tetragonal CH2 geometry
+# 3. H out of plane of its 3 neighbors (should be rare and not in AA)
+def get_coefficients(rh, a0, a1, a2, ih2, idealize, sites_cart, typeh):
+  r0 = matrix.col(sites_cart[a0.iseq])
+  r1 = matrix.col(sites_cart[a1.iseq])
+  r2 = matrix.col(sites_cart[a2.iseq])
+  uh0 = (rh - r0).normalize()
+  u10 = (r1 - r0).normalize()
+  u20 = (r2 - r0).normalize()
+  if idealize:
+    alpha0 = math.radians(a0.angle_ideal)
+    alpha1 = math.radians(a1.angle_ideal)
+    alpha2 = math.radians(a2.angle_ideal)
+    c0, c1, c2 = math.cos(alpha0), math.cos(alpha1), math.cos(alpha2)
+  else:
+    alpha0 = (u10).angle(u20)
+    alpha1 = (u10).angle(uh0)
+    alpha2 = (uh0).angle(u20)
+    c0 = (u10).dot(u20)
+    c1 = (u10).dot(uh0)
+    c2 = (uh0).dot(u20)
+  sumang = alpha0 + alpha1 + alpha2
+  denom = (1.0-c0**2)
+  if(denom==0):
+    raise RuntimeError(
+      "Denominator zero: (1-c0*c0) in get_h_parameterization.")
+  a = (c1-c0*c2)/(1-c0*c0)
+  b = (c2-c0*c1)/(1-c0*c0)
+  # check if H, A0, A1, A2 are in a plane
+  if (sumang < (2*math.pi + 0.04) and (sumang > 2*math.pi - 0.04)):
+    h = None
+  else:
+    # two tetragonal geometry: e.g. CH2 group
+    if (ih2 is not None):
+      rh2 = matrix.col(sites_cart[ih2.iseq])
+      uh02 = (rh2 - r0).normalize()
+      if idealize:
+        h = math.radians(ih2.angle_ideal) * 0.5
+      else:
+        h = (uh0).angle(uh02) * 0.5
+      #test if vector v points to same 'side' as uh0
+      if((u10.cross(u20)).dot(uh0) < 0):
+        h =  -h
+    else:
+    # if H is out of plane, but not in tetrahedral geometry
+      root = 1-c1*c1-c2*c2-c0*c0+2*c0*c1*c2
+      if(root < 0):
+        raise RuntimeError(
+          "Expression in square root < 0 in get_h_parameterization.")
+      denom = math.sin(alpha0)
+      if(denom==0):
+        raise RuntimeError(
+          "Denominator zero: sin(alpha0)in get_h_parameterization.")
+      cz = (math.sqrt(1-c1*c1-c2*c2-c0*c0+2*c0*c1*c2))/math.sin(alpha0)
+      h = cz
+      #test if vector v points to same 'side' as uh0
+      if((u10.cross(u20)).dot(uh0) < 0):
+        h = -h
+  return sumang, a, b, h
 
-# for every H atom, determine type of bond
+# obtain coefficients for tetragonal H (such as HA) using Cramer's rule
+def get_coefficients_alg3(rh, a0, a1, a2, a3, idealize, sites_cart):
+  r0 = matrix.col(sites_cart[a0.iseq])
+  r1 = matrix.col(sites_cart[a1.iseq])
+  r2 = matrix.col(sites_cart[a2.iseq])
+  r3 = matrix.col(sites_cart[a3.iseq])
+  uh0 = (rh - r0).normalize()
+  u10 = (r1 - r0).normalize()
+  u20 = (r2 - r0).normalize()
+  u30 = (r3 - r0).normalize()
+  if idealize:
+    alpha0 = math.radians(a1.angle_ideal)
+    alpha1 = math.radians(a2.angle_ideal)
+    alpha2 = math.radians(a3.angle_ideal)
+    c1, c2, c3 = math.cos(alpha0), math.cos(alpha1), math.cos(alpha2)
+    omega0 = math.radians(a0.angle_ideal[0])
+    omega1 = math.radians(a0.angle_ideal[1])
+    omega2 = math.radians(a0.angle_ideal[2])
+    w12, w23, w13 = math.cos(omega0), math.cos(omega1), math.cos(omega2)
+  else:
+    c1 = (uh0).dot(u10)
+    c2 = (uh0).dot(u20)
+    c3 = (uh0).dot(u30)
+    w12 = (u10).dot(u20)
+    w23 = (u20).dot(u30)
+    w13 = (u10).dot(u30)
+  matrix_d = matrix.sqr([
+    1,   w12, w13,
+    w12, 1,   w23,
+    w13, w23, 1   ])
+  #
+  matrix_x = matrix.sqr([
+    c1, w12, w13,
+    c2, 1,   w23,
+    c3, w23, 1   ])
+  #
+  matrix_y = matrix.sqr([
+    1,   c1,  w13,
+    w12, c2,  w23,
+    w13, c3,  1   ])
+  #
+  matrix_z = matrix.sqr([
+    1,   w12,  c1,
+    w12, 1,    c2,
+    w13, w23,  c3 ])
+  if(matrix_d.determinant()==0):
+    raise RuntimeError(
+      "Denominator zero: matrix_d in get_h_parameterization.")
+  a = matrix_x.determinant()/matrix_d.determinant()
+  b = matrix_y.determinant()/matrix_d.determinant()
+  c = matrix_z.determinant()/matrix_d.determinant()
+  return a, b, c
+
+
+# for every H atom, determine the type of bond
 def get_h_parameterization(connectivity, sites_cart, idealize):
   h_parameterization = {}
   for ih in connectivity.keys():
+    # if entry exists already, skip it
+    if ih in h_parameterization:
+      continue
     a0 = connectivity[ih][0]
     count_H, reduced_neighbs = a0.count_H, a0.reduced_neighbs
     n_red_neigbs = len(reduced_neighbs)
@@ -72,34 +191,28 @@
       dist_h = a0.dist_ideal
     else:
       dist_h = (r0 - rh).length()
-  # case 2a and 2b, case 3
-    if(n_red_neigbs == 2 or
-      (n_red_neigbs == 3 and count_H == 0)):
+  # alg2a, 2tetra, 2neigbs
+    if(n_red_neigbs == 2):
       a1, a2  = reduced_neighbs[0], reduced_neighbs[1]
-      r1 = matrix.col(sites_cart[a1.iseq])
-      r2 = matrix.col(sites_cart[a2.iseq])
-      uh0 = (rh - r0).normalize()
-      u10 = (r1 - r0).normalize()
-      u20 = (r2 - r0).normalize()
-      if idealize:
-        alpha0 = math.radians(a0.angle_ideal)
-        alpha1 = math.radians(a1.angle_ideal)
-        alpha2 = math.radians(a2.angle_ideal)
-        c0, c1, c2 = math.cos(alpha0), math.cos(alpha1), math.cos(alpha2)
+      # if H is second neighbor, gets its index
+      if (count_H == 1):
+        hlist = connectivity[ih][1]
+        hlist.remove(a1)
+        hlist.remove(a2)
+        if hlist:
+          ih2 = (hlist[0])
+          i_h2 = (hlist[0]).iseq
       else:
-        alpha0 = (u10).angle(u20)
-        alpha1 = (u10).angle(uh0)
-        alpha2 = (uh0).angle(u20)
-        c0 = (u10).dot(u20)
-        c1 = (u10).dot(uh0)
-        c2 = (uh0).dot(u20)
-      sumang = alpha0 + alpha1 + alpha2
-      denom = (1.0-c0**2)
-      if(denom==0):
-        raise RuntimeError(
-          "Denominator zero: (1-c0*c0) in get_h_parameterization.")
-      a = (c1-c0*c2)/(1-c0*c0)
-      b = (c2-c0*c1)/(1-c0*c0)
+        ih2 = None
+      sumang, a, b, h = get_coefficients(
+        rh         = rh,
+        a0         = a0,
+        a1         = a1,
+        a2         = a2,
+        ih2        = ih2,
+        idealize   = idealize,
+        sites_cart = sites_cart,
+        typeh      = 'alg2')
       h_parameterization[ih] = parameterization_info(
         a0     = a0.iseq,
         a1     = a1.iseq,
@@ -107,82 +220,100 @@
         a      = a,
         b      = b,
         dist_h = dist_h)
-      #if ((sumang < 361 and sumang > 359) and idealize == True ):
-      #if (0):
-      #if (sumang < 361 and sumang > 359):
+      # alg2a
       if (sumang < (2*math.pi + 0.05) and (sumang > 2*math.pi - 0.05)):
         h_parameterization[ih].htype = 'flat_2neigbs'
       else:
-        root = 1-c1*c1-c2*c2-c0*c0+2*c0*c1*c2
-        if(root < 0):
-          raise RuntimeError(
-            "Expression in square root < 0 in get_h_parameterization.")
-        denom = math.sin(alpha0)
-        if(denom==0):
-          raise RuntimeError(
-            "Denominator zero: sin(alpha0)in get_h_parameterization.")
-        cz = (math.sqrt(1-c1*c1-c2*c2-c0*c0+2*c0*c1*c2))/math.sin(alpha0)
-        h = cz/math.sin(alpha0)
-        #test if vector v points to same 'side' as uh0
-        if((u10.cross(u20)).dot(uh0) < 0):
-          h = -h
-        h_parameterization[ih].h = h
-        if (n_red_neigbs == 2): # case 2b
+        # 2 tetragonal geometry
+        if (count_H == 1):
+          h_parameterization[ih].htype = '2tetra'
+          h_parameterization[ih].alpha = h
+          h_parameterization[i_h2] = parameterization_info(
+            a0     = a0.iseq,
+            a1     = a1.iseq,
+            a2     = a2.iseq,
+            a      = a,
+            b      = b,
+            alpha  = -h,
+            dist_h = dist_h,
+            htype  = '2tetra')
+        else:
+          # 2neigbs
+          h_parameterization[ih].h = h
           h_parameterization[ih].htype = '2neigbs'
-        elif (n_red_neigbs == 3): # case 3
-          h_parameterization[ih].htype = '3neigbs'
-        #if(count_H == 1):
-        #  print h_parameterization[ih].htype
-    # case 1a
+    # tetragonal geometry: 3neigbs
+    elif (n_red_neigbs == 3 and count_H == 0):
+      a1, a2, a3  = reduced_neighbs[0], reduced_neighbs[1], reduced_neighbs[2]
+      a, b, h = get_coefficients_alg3(
+        rh         = rh,
+        a0         = a0,
+        a1         = a1,
+        a2         = a2,
+        a3         = a3,
+        idealize   = idealize,
+        sites_cart = sites_cart)
+      h_parameterization[ih] = parameterization_info(
+        a0     = a0.iseq,
+        a1     = a1.iseq,
+        a2     = a2.iseq,
+        a3     = a3.iseq,
+        a      = a,
+        b      = b,
+        h      = h,
+        dist_h = dist_h,
+        htype  = '3neigbs')
+    # alg1a: X-H2 planar groups, such as in ARG, ASN, GLN
     elif(n_red_neigbs == 1 and count_H == 1 and len(connectivity[ih][2])==2):
-      neigbs_14 = connectivity[ih][2]
       a1 = reduced_neighbs[0]
-      b1, b2 = neigbs_14[0], neigbs_14[1]
       r1 = matrix.col(sites_cart[a1.iseq])
-      rb1 = matrix.col(sites_cart[b1.iseq])
-      rb2 = matrix.col(sites_cart[b2.iseq])
-      # chose 1-4 neighbor which is closer - important!
-      if((rh-rb2).length() < (rh-rb1).length()):
-        neigbs_14[0], neigbs_14[1] = neigbs_14[1], neigbs_14[0]
-      rb1 = matrix.col(sites_cart[neigbs_14[0].iseq])
-      r2 = r0 + (r1 - rb1)
+      hlist = connectivity[ih][1]
+      hlist.remove(a1)
+      ih_2 = hlist[0].iseq
+      if(a0.dihe_list is None):
+        continue
+      iseq_b1 = a0.dihe_list[3]
+      dihedral = dihedral_angle(
+        sites=[sites_cart[iseq_b1], sites_cart[a1.iseq],
+        sites_cart[a0.iseq],sites_cart[ih]])
+      #print 'dihedrals', a0.dihedral, dihedral, a0.dihedral_ideal
+      rb1 = matrix.col(sites_cart[iseq_b1])
       uh0 = (rh - r0).normalize()
       u10 = (r1 - r0).normalize()
-      u20 = (r2 - r0).normalize()
-      alpha0 = (u10).angle(u20)
-      alpha1 = (u10).angle(uh0)
-      alpha2 = (uh0).angle(u20)
-      c0, c1, c2 = math.cos(alpha0), math.cos(alpha1), math.cos(alpha2)
-      denom = (1-c0*c0)
-      if(denom==0):
-        raise RuntimeError(
-          "Denominator zero: (1-c0*c0) in get_h_parameterization.")
-      a = (c1-c0*c2)/(1-c0*c0)
-      b = (c2-c0*c1)/(1-c0*c0)
-      root = 1-c1*c1-c2*c2-c0*c0+2*c0*c1*c2
-      if(root < 0):
-        raise RuntimeError(
-          "Expression in square root < 0 in get_h_parameterization.")
-      denom = math.sin(alpha0)
-      if(denom==0):
-        raise RuntimeError(
-          "Denominator zero: sin(alpha0)in get_h_parameterization.")
-      cz = (math.sqrt(1-c1*c1-c2*c2-c0*c0+2*c0*c1*c2))/math.sin(alpha0)
-      h = cz/math.sin(alpha0)
-      # test if vector v points to same 'side' as uh0
-      if((u10.cross(u20)).dot(uh0) < 0):
-        h = -h
+      if idealize:
+        alpha = math.radians(a1.angle_ideal)
+        phi = math.radians(a0.dihedral_ideal)
+        #phi = dihedral
+      else:
+        alpha = (u10).angle(uh0)
+        #phi = a0.dihedral
+        phi = dihedral
+      u1 = (r0 - r1).normalize()
+      rb10 = rb1 - r1
+      u2 = (rb10 - ((rb10).dot(u10)) * u10).normalize()
+      u3 = u1.cross(u2)
       h_parameterization[ih] = parameterization_info(
         htype  = 'alg1a',
         a0     = a0.iseq,
         a1     = a1.iseq,
-        a2     = neigbs_14[0].iseq,
-        a      = a,
-        b      = b,
-        h      = h,
+        a2     = iseq_b1,
+        phi    = phi,
+        n      = 0,
+        alpha  = alpha,
         dist_h = dist_h)
+      h_parameterization[ih_2] = parameterization_info(
+        htype  = 'alg1a',
+        a0     = a0.iseq,
+        a1     = a1.iseq,
+        a2     = iseq_b1,
+        phi    = phi+math.pi,
+        n      = 0,
+        alpha  = alpha,
+        dist_h = dist_h)
     # case 1b
+# a0.dihedral = dihedral angle between angle ideal and actual position
     elif(n_red_neigbs == 1 and (count_H == 0 or count_H ==2)):
+      if(count_H == 2 and a0.dihe_list is None):
+        continue
       a1 = reduced_neighbs[0]
       sec_neigbs = connectivity[ih][2]
       b1 = sec_neigbs[0]
@@ -190,13 +321,18 @@
       rb1 = matrix.col(sites_cart[b1.iseq])
       uh0 = (rh - r0).normalize()
       u10 = (r1 - r0).normalize()
+      dihedral = dihedral_angle(
+        sites=[sites_cart[ih], sites_cart[a0.iseq],
+        sites_cart[a1.iseq],sites_cart[b1.iseq]])
       if idealize:
         alpha = math.radians(a1.angle_ideal)
+        #phi = math.radians(a0.dihedral_ideal)
+        #allow for rotation even for idealize = True
+        #phi = a0.dihedral
+        phi = dihedral
       else:
         alpha = (u10).angle(uh0)
-      phi = dihedral_angle(
-        sites=[sites_cart[ih], sites_cart[a0.iseq],
-        sites_cart[a1.iseq],sites_cart[b1.iseq]])
+        phi = dihedral
       u1 = (r0 - r1).normalize()
       rb10 = rb1 - r1
       u2 = (rb10 - ((rb10).dot(u1)) * u1).normalize()
@@ -207,8 +343,33 @@
         a1     = a1.iseq,
         a2     = b1.iseq,
         phi    = phi,
+        n      = 0,
         alpha  = alpha,
         dist_h = dist_h)
+      if (count_H == 2):
+        h_parameterization[ih].htype = 'prop'
+        hlist = connectivity[ih][1]
+        hlist.remove(a1)
+        # TO DO: Can the order be reversed? To be kept in mind!!
+        ih_2, ih_3 = hlist[0].iseq, hlist[1].iseq
+        h_parameterization[ih_2] = parameterization_info(
+          htype  = 'prop',
+          a0     = a0.iseq,
+          a1     = a1.iseq,
+          a2     = b1.iseq,
+          phi    = phi,
+          n      = 1,
+          alpha  = alpha,
+          dist_h = dist_h)
+        h_parameterization[ih_3] = parameterization_info(
+          htype  = 'prop',
+          a0     = a0.iseq,
+          a1     = a1.iseq,
+          a2     = b1.iseq,
+          phi    = phi,
+          n      = 2,
+          alpha  = alpha,
+          dist_h = dist_h)
     else:
       h_parameterization[ih] = parameterization_info(
         htype  = 'unk',
@@ -227,32 +388,55 @@
     a, b = para_info.a, para_info.b
     r2 = matrix.col(sites_cart[para_info.a2])
     u10, u20 = (r1 - r0).normalize(), (r2 - r0).normalize()
-    uh0 = (a * u10 + b * u20)
+    length = math.sqrt(a*a + b*b + 2*a*b*(u10).dot(u20))
+    if(length==0):
+      raise RuntimeError("Denominator zero: length in generate_H_positions")
+    uh0 = (a * u10 + b * u20)/length
     rH_gen = r0 + dh * uh0
     deltaH = (rH_gen - matrix.col(sites_cart[ih])).length()
-  # alg2b and alg3
-  elif (para_info.htype == '2neigbs' or para_info.htype == '3neigbs'):
+  # 2 neigbs
+  elif (para_info.htype == '2neigbs'):
     a, b, h = para_info.a, para_info.b, para_info.h
     r2 = matrix.col(sites_cart[para_info.a2])
     u10, u20 = (r1 - r0).normalize(), (r2 - r0).normalize()
-    v = u10.cross(u20)
-    uh0 = (a * u10 + b * u20 + h * v)
+    v0 = (u10.cross(u20)).normalize()
+    rh0 = (a * u10 + b * u20 + h * v0)
+    length = math.sqrt(rh0.dot(rh0))
+    if(length==0):
+      raise RuntimeError("Denominator zero: length in generate_H_positions")
+    uh0 = rh0/length
     rH_gen = r0 + dh * uh0
     deltaH = (rH_gen - matrix.col(sites_cart[ih])).length()
-  # alg1a
-  elif (para_info.htype == 'alg1a'):
+  # 2tetrahedral
+  elif (para_info.htype == '2tetra'):
+    a, b, delta = para_info.a, para_info.b, para_info.alpha
+    r2 = matrix.col(sites_cart[para_info.a2])
+    u10, u20 = (r1 - r0).normalize(), (r2 - r0).normalize()
+    v0 = (u10.cross(u20)).normalize()
+    d0 = (a * u10 + b * u20).normalize()
+    rH_gen = r0 + dh * (math.cos(delta) * d0 + math.sin(delta) * v0)
+    deltaH = (rH_gen - matrix.col(sites_cart[ih])).length()
+  # tetragonal alg3
+  elif (para_info.htype == '3neigbs'):
     a, b, h = para_info.a, para_info.b, para_info.h
-    rb1 = matrix.col(sites_cart[para_info.a2])
-    r2 = r0 + (r1 - rb1)
-    u10, u20 = (r1 - r0).normalize(), (r2 - r0).normalize()
-    v = u10.cross(u20)
-    uh0 = (a * u10 + b * u20 + h * v)
+    r2 = matrix.col(sites_cart[para_info.a2])
+    r3 = matrix.col(sites_cart[para_info.a3])
+    u10 = (r1 - r0).normalize()
+    u20 = (r2 - r0).normalize()
+    u30 = (r3 - r0).normalize()
+    rh0 = (a*u10 + b*u20 + h*u30)
+    length = math.sqrt(rh0.dot(rh0))
+    if(length==0):
+      raise RuntimeError("Denominator zero: length in generate_H_positions")
+    uh0 = rh0/length
     rH_gen = r0 + dh * uh0
     deltaH = (rH_gen - matrix.col(sites_cart[ih])).length()
-#
-  elif (para_info.htype == 'alg1b'):
+# alg1b or alg1a or propeller group
+  elif (para_info.htype in ['alg1b', 'alg1a', 'prop']):
     rb1 = matrix.col(sites_cart[para_info.a2])
-    phi = para_info.phi
+    n = para_info.n
+    phi = para_info.phi + n*2*math.pi/3
+    #phi = para_info.phi
     alpha = para_info.alpha
     salpha = math.sin(alpha)
     calpha = math.cos(alpha)
@@ -270,7 +454,6 @@
     distance = deltaH,
     rH_gen   = rH_gen)
 
-
 def run(args, out=sys.stdout):
   log = multi_out()
   log.register("stdout", out)
@@ -317,18 +500,21 @@
   bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
     sites_cart = xray_structure.sites_cart())
   angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
+  dihedral_proxies = restraints_manager.geometry.dihedral_proxies
   hd_selection = xray_structure.hd_selection()
   names = list(pdb_hierarchy.atoms().extract_name())
   sites_cart = xray_structure.sites_cart()
   #scatterers = xray_structure.scatterers()
-
   atoms_list = list(pdb_hierarchy.atoms_with_labels())
 
+  idealize = True
+
   print >>log, '\nNow determining connectivity table for H atoms...'
   connectivity = hydrogen_connectivity.determine_H_neighbors(
     geometry_restraints   = geometry_restraints,
     bond_proxies          = bond_proxies_simple,
     angle_proxies         = angle_proxies,
+    dihedral_proxies      = dihedral_proxies,
     hd_selection          = hd_selection,
     sites_cart            = sites_cart)
 
@@ -336,7 +522,7 @@
   h_parameterization = get_h_parameterization(
     connectivity   = connectivity,
     sites_cart     = sites_cart,
-    idealize       = True)
+    idealize       = idealize)
 
   print >>log, '\nNow reconstructing H atoms...'
   long_distance_list = []
@@ -351,13 +537,13 @@
     if(h_obj.distance is not None):
       print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
         residue, 'distance:', h_obj.distance
-      if(h_obj.distance > 0.05):
+      if(h_obj.distance > 0.03):
         long_distance_list.append(ih)
     else:
       print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', residue
       unk_list.append(ih)
 
-  # some informative output for residues with unknown algorithm
+  # list output for residues with unknown algorithm
   if unk_list:
     print >>log, '*'*79
     print >>log, 'Warning: The following atoms where not assigned an H type'
@@ -368,8 +554,7 @@
         'chain', atoms_list[ih].chain_id
     print >>log, '*'*79
 
-  # some informative output for residues where position is NOT reproduced
-  # -> wronlgy assigned
+  # list output for residues where position is not reproduced
   if long_distance_list:
     print >>log, '*'*79
     print >>log, 'Warning: The position of the following H atoms was not reproduced'
@@ -380,13 +565,23 @@
         sites_cart        = sites_cart,
         ih                = ih,
         para_info         = hp)
-      if(h_obj.distance is not None and h_obj.distance > 0.05):
+      if(h_obj.distance is not None and h_obj.distance > 0.0014):
         print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
           residue, 'chain', atoms_list[ih].chain_id, 'distance:', h_obj.distance
+      sites_cart[ih] = h_obj.rH_gen
+  xray_structure.set_sites_cart(sites_cart)
+  pdb_hierarchy.adopt_xray_structure(xray_structure)
+  print pdb_filename
+  STOP()
+  pdb_hierarchy.write_pdb_file(
+    file_name        = pdb_filename+"newH.pdb",
+    crystal_symmetry = xray_structure.crystal_symmetry())
   print >>log, '*'*79
 
   #for ih in h_parameterization.keys():
   #  hp = h_parameterization[ih]
+  #  #if(ih != 220):
+  #  #  continue
   #  print 'htype = ', hp.htype, 'a0 = ', hp.a0, 'a1 = ', hp.a1, 'a2 = ', hp.a2, \
   #    'a = ', hp.a, 'b = ', hp.b, 'h = ', hp.h, 'phi = ', hp.phi, \
   #    'alpha = ', hp.alpha, 'dist_h =', hp.dist_h

Added: trunk/mmtbx/hydrogens/tst_h_connectivity.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_h_connectivity.py	                        (rev 0)
+++ trunk/mmtbx/hydrogens/tst_h_connectivity.py	2016-10-05 22:42:31 UTC (rev 25554)
@@ -0,0 +1,152 @@
+from __future__ import division
+import time
+
+import mmtbx.monomer_library.server
+import mmtbx.monomer_library.pdb_interpretation
+import mmtbx.model
+from mmtbx import monomer_library
+from cctbx import geometry_restraints
+import hydrogen_connectivity
+#from mmtbx import hydrogens
+
+pdb_str = """\
+CRYST1   17.955   13.272   13.095  90.00  90.00  90.00 P 1
+ATOM      1  N   TYR A 139      10.241   7.920   5.000  1.00 10.00           N
+ATOM      2  CA  TYR A 139      10.853   7.555   6.271  1.00 10.00           C
+ATOM      3  C   TYR A 139      12.362   7.771   6.227  1.00 10.00           C
+ATOM      4  O   TYR A 139      12.955   8.272   7.181  1.00 10.00           O
+ATOM      5  CB  TYR A 139      10.540   6.098   6.617  1.00 10.00           C
+ATOM      6  CG  TYR A 139       9.063   5.805   6.749  1.00 10.00           C
+ATOM      7  CD1 TYR A 139       8.316   5.391   5.654  1.00 10.00           C
+ATOM      8  CD2 TYR A 139       8.414   5.943   7.969  1.00 10.00           C
+ATOM      9  CE1 TYR A 139       6.966   5.122   5.770  1.00 10.00           C
+ATOM     10  CE2 TYR A 139       7.064   5.676   8.095  1.00 10.00           C
+ATOM     11  CZ  TYR A 139       6.345   5.266   6.993  1.00 10.00           C
+ATOM     12  OH  TYR A 139       5.000   5.000   7.113  1.00 10.00           O
+ATOM     13  HA  TYR A 139      10.480   8.127   6.960  1.00 10.00           H
+ATOM     14  HB2 TYR A 139      10.915   5.524   5.931  1.00 10.00           H
+ATOM     15  HB3 TYR A 139      10.982   5.870   7.450  1.00 10.00           H
+ATOM     16  HD1 TYR A 139       8.732   5.293   4.828  1.00 10.00           H
+ATOM     17  HD2 TYR A 139       8.896   6.220   8.714  1.00 10.00           H
+ATOM     18  HE1 TYR A 139       6.479   4.845   5.028  1.00 10.00           H
+ATOM     19  HE2 TYR A 139       6.643   5.772   8.919  1.00 10.00           H
+ATOM     20  HH  TYR A 139       4.759   5.128   7.907  1.00 10.00           H
+TER
+END
+"""
+
+#----------------------------------------------------
+# This test checks for residue Tyr (pdb_str above):
+# - if all bonds involving H atoms are recognized
+# - if all angles involving H atoms are recognized
+# - if 3rd neighbors of HH atom are correctly found
+#----------------------------------------------------
+
+def exercise():
+  mon_lib_srv = monomer_library.server.server()
+  ener_lib = monomer_library.server.ener_lib()
+  processed_pdb_file = monomer_library.pdb_interpretation.process(
+    mon_lib_srv    = mon_lib_srv,
+    ener_lib       = ener_lib,
+    file_name      = None,
+    raw_records    = pdb_str,
+    force_symmetry = True)
+  pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy
+  xray_structure = processed_pdb_file.xray_structure()
+
+  geometry_restraints = processed_pdb_file.geometry_restraints_manager(
+    show_energies = False)
+  restraints_manager = mmtbx.restraints.manager(
+    geometry      = geometry_restraints,
+    normalization = False)
+
+  sites_cart = xray_structure.sites_cart()
+
+  bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
+      sites_cart = sites_cart)
+  angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
+  dihedral_proxies = restraints_manager.geometry.dihedral_proxies
+  hd_selection = xray_structure.hd_selection()
+
+  # determine values with code to be tested
+  connectivity = hydrogen_connectivity.determine_H_neighbors(
+    geometry_restraints   = geometry_restraints,
+    bond_proxies          = bond_proxies_simple,
+    angle_proxies         = angle_proxies,
+    dihedral_proxies      = dihedral_proxies,
+    hd_selection          = hd_selection,
+    sites_cart            = sites_cart)
+
+  bond_list = {}
+  angle_list = {}
+  third_nb_list = {}
+  for ih in connectivity.keys():
+    a0 = (connectivity[ih][0])
+    bond_list[ih]=[a0.iseq, a0.dist_ideal]
+    for atom in connectivity[ih][1]:
+      helper = tuple(sorted([ih, a0.iseq, atom.iseq]))
+      angle_list[helper]=atom.angle_ideal
+    if(len(connectivity[ih])==3):
+      third_nb_list[ih]=[]
+      for atom in connectivity[ih][2]:
+        third_nb_list[ih].append(atom.iseq)
+#      print third_nb_list[ih]
+
+#-----------------------------------------------------------------------------
+# This is useful to keep for debugging: human readable output of connectivity
+#-----------------------------------------------------------------------------
+#  for ih in connectivity.keys():
+#    if(len(connectivity[ih])==3):
+#      string = (" ".join([names[p.iseq] for p in connectivity[ih][2]]))
+#    else:
+#      string = 'n/a'
+#    print  names[ih],': ', names[(connectivity[ih][0][0]).iseq], \
+#      ',', (" ".join([names[p.iseq] for p in connectivity[ih][1]])), \
+#      ',', string
+#-----------------------------------------------------------------------------
+
+# determine values directly from pdb_str
+  model = mmtbx.model.manager(
+    restraints_manager = restraints_manager,
+    xray_structure     = xray_structure,
+    pdb_hierarchy      = pdb_hierarchy)
+  bond_ctrl = {}
+  for i in model.xh_connectivity_table():
+    bond_ctrl[i[1]]=[i[0],i[3]]
+
+# We know the angles beforehand
+  angles = [
+    (4, 1, 12),
+    (2, 1, 12),
+    (0, 1, 12),
+    (13, 4, 14),
+    (5, 4, 14),
+    (5, 4, 13),
+    (1, 4, 13),
+    (1, 4, 14),
+    (8, 6, 15),
+    (5, 6, 15),
+    (9, 7, 16),
+    (5, 7, 16),
+    (10, 8, 17),
+    (6, 8, 17),
+    (10, 11, 19),
+    (7, 9, 18),
+    (10, 9, 18)]
+
+  angle_ctrl = {}
+  for ap in angle_proxies:
+    if(ap.i_seqs in angles):
+      angle_ctrl[tuple(sorted(list(ap.i_seqs)))]=ap.angle_ideal
+
+# HH needs also third neighbors:
+  third_nb_ctrl = {19: [8, 9]}
+
+  assert (bond_list == bond_ctrl), '1-2 neighbors and distance_ideal are wrong'
+  assert (angle_list == angle_ctrl), '1-3 neighbors and angle_ideal are wrong'
+  assert (third_nb_list == third_nb_ctrl), '1-4 neighbors are wrong'
+
+if (__name__ == "__main__"):
+  t0 = time.time()
+  exercise()
+  print "OK. Time: %8.3f"%(time.time()-t0)

Modified: trunk/mmtbx/hydrogens/tst_h_parametrization_1.py
===================================================================
--- trunk/mmtbx/hydrogens/tst_h_parametrization_1.py	2016-10-05 22:01:45 UTC (rev 25553)
+++ trunk/mmtbx/hydrogens/tst_h_parametrization_1.py	2016-10-05 22:42:31 UTC (rev 25554)
@@ -41,24 +41,27 @@
   bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
       sites_cart = sites_cart)
   angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
+  dihedral_proxies = restraints_manager.geometry.dihedral_proxies
   hd_selection = xray_structure.hd_selection()
 
   connectivity = hydrogen_connectivity.determine_H_neighbors(
     geometry_restraints   = geometry_restraints,
     bond_proxies          = bond_proxies_simple,
     angle_proxies         = angle_proxies,
+    dihedral_proxies      = dihedral_proxies,
     hd_selection          = hd_selection,
     sites_cart            = sites_cart)
-
+  idealize = True
   h_parameterization = hydrogen_parametrization.get_h_parameterization(
     connectivity   = connectivity,
     sites_cart     = sites_cart,
-    idealize       = False)
+    idealize       = idealize)
 
 # There are 152 H atoms in the pdb_string, check if all of them are recognized
   assert (len(h_parameterization.keys()) == 152), 'Not all H atoms are parameterized'
 
-# For each H atom, check if distance compared to input model is not > 0.001
+# For each H atom, check if distance compared to input model is not > 0.05
+# 0.0014 = uncertainty for distance if uncertainty of coordinate = 0.001
   n_unk = 0
   type_list = []
   for ih in h_parameterization.keys():
@@ -66,14 +69,15 @@
     hp = h_parameterization[ih]
     type_list.append(hp.htype)
     h_obj = hydrogen_parametrization.generate_H_positions(
-      sites_cart        = sites_cart,
-      ih                = ih,
-      para_info         = hp)
-    assert (h_obj.distance < 0.001), 'distance too large: %s  atom: %s (%s) residue: %s ' \
+      sites_cart = sites_cart,
+      ih         = ih,
+      para_info  = hp)
+    assert (h_obj.distance < 0.05), 'distance too large: %s  atom: %s (%s) residue: %s ' \
       % (hp.htype, names[ih], ih, residue)
     if(hp.htype == 'unk'):
       n_unk = n_unk + 1
 
+# KEEP: useful for debugging
   #for ih in h_parameterization.keys():
   #  hp = h_parameterization[ih]
   #  print "'"+hp.htype+"'"+',',
@@ -81,25 +85,23 @@
   assert(n_unk == 0), 'Some H atoms are not recognized'
 
   for type1, type2 in zip(type_list, type_list_known):
-    #print type1, type2
     assert (type1 == type2)
 
 #-----------------------------------------------------------------------------
 # This is useful to keep for debugging
 #-----------------------------------------------------------------------------
-  #for ih in h_parameterization.keys():
-  #  hp = h_parameterization[ih]
-    #if(h_obj.distance is not None):
-    #  print hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
-    #    residue, 'distance:', h_obj.distance
-    #else:
-    #  print hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', residue
-  #for ih in h_parameterization.keys():
-  #  hp = h_parameterization[ih]
-  #  print 'htype = ', hp.htype, 'a0 = ', hp.a0, 'a1 = ', hp.a1, 'a2 = ', hp.a2, \
-  #    'a = ', hp.a, 'b = ', hp.b, 'h = ', hp.h, 'chi = ', hp.chi, 'eps = ', hp.eps, \
-  #    'alpha = ', hp.alpha, 'dist_h =', hp.dist_h
-
+#  for ih in h_parameterization.keys():
+#    residue = atoms_list[ih].resseq
+#    hp = h_parameterization[ih]
+#    h_obj = hydrogen_parametrization.generate_H_positions(
+#      sites_cart        = sites_cart,
+#      ih                = ih,
+#      para_info         = hp)
+#    if(h_obj.distance is not None):
+#      print hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
+#        residue, 'distance:', h_obj.distance
+#    else:
+#      print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', residue
 #-----------------------------------------------------------------------------
 
 # Ideal amino acids
@@ -108,379 +110,357 @@
 SCALE1      0.033333  0.000000  0.000000        0.00000
 SCALE2      0.000000  0.033333  0.000000        0.00000
 SCALE3      0.000000  0.000000  0.033333        0.00000
-ATOM      1  N   ARG A   2       3.250  19.540  25.752  1.00  0.00           N
-ATOM      2  CA  ARG A   2       3.908  20.348  24.733  1.00  0.00           C
-ATOM      3  C   ARG A   2       5.395  20.503  25.033  1.00  0.00           C
-ATOM      4  O   ARG A   2       5.777  21.119  26.029  1.00  0.00           O
-ATOM      5  CB  ARG A   2       3.246  21.724  24.629  1.00  0.00           C
-ATOM      6  CG  ARG A   2       3.853  22.627  23.568  1.00  0.00           C
-ATOM      7  CD  ARG A   2       3.152  23.975  23.522  1.00  0.00           C
-ATOM      8  NE  ARG A   2       3.720  24.853  22.503  1.00  0.00           N
-ATOM      9  CZ  ARG A   2       3.291  26.086  22.257  1.00  0.00           C
-ATOM     10  NH1 ARG A   2       2.285  26.593  22.956  1.00  0.00           N1+
-ATOM     11  NH2 ARG A   2       3.868  26.814  21.310  1.00  0.00           N
-ATOM      0  HA  ARG A   2       3.814  19.890  23.883  1.00  0.00           H
-ATOM      0  HB2 ARG A   2       2.303  21.605  24.437  1.00  0.00           H
-ATOM      0  HB3 ARG A   2       3.306  22.167  25.490  1.00  0.00           H
-ATOM      0  HG2 ARG A   2       4.796  22.757  23.752  1.00  0.00           H
-ATOM      0  HG3 ARG A   2       3.790  22.198  22.700  1.00  0.00           H
-ATOM      0  HD2 ARG A   2       2.208  23.842  23.343  1.00  0.00           H
-ATOM      0  HD3 ARG A   2       3.219  24.403  24.390  1.00  0.00           H
-ATOM      0  HE  ARG A   2       4.374  24.553  22.032  1.00  0.00           H
-ATOM      0 HH11 ARG A   2       1.909  26.123  23.570  1.00  0.00           H
-ATOM      0 HH12 ARG A   2       2.009  27.391  22.795  1.00  0.00           H
-ATOM      0 HH21 ARG A   2       4.521  26.488  20.855  1.00  0.00           H
-ATOM      0 HH22 ARG A   2       3.590  27.612  21.152  1.00  0.00           H
-ATOM     12  N   HIS A   3       6.232  19.940  24.165  1.00  0.00           N
-ATOM     13  CA  HIS A   3       7.680  19.999  24.307  1.00  0.00           C
-ATOM     14  C   HIS A   3       8.283  20.588  23.041  1.00  0.00           C
-ATOM     15  O   HIS A   3       7.916  20.190  21.930  1.00  0.00           O
-ATOM     16  CB  HIS A   3       8.266  18.611  24.580  1.00  0.00           C
-ATOM     17  CG  HIS A   3       7.752  17.974  25.834  1.00  0.00           C
-ATOM     18  ND1 HIS A   3       8.265  18.264  27.080  1.00  0.00           N
-ATOM     19  CD2 HIS A   3       6.771  17.062  26.033  1.00  0.00           C
-ATOM     20  CE1 HIS A   3       7.622  17.558  27.993  1.00  0.00           C
-ATOM     21  NE2 HIS A   3       6.711  16.820  27.384  1.00  0.00           N
-ATOM      0  H   HIS A   3       5.969  19.508  23.469  1.00  0.00           H
-ATOM      0  HA  HIS A   3       7.897  20.564  25.065  1.00  0.00           H
-ATOM      0  HB2 HIS A   3       8.067  18.032  23.828  1.00  0.00           H
-ATOM      0  HB3 HIS A   3       9.232  18.682  24.636  1.00  0.00           H
-ATOM      0  HD2 HIS A   3       6.238  16.673  25.378  1.00  0.00           H
-ATOM      0  HE1 HIS A   3       7.783  17.577  28.909  1.00  0.00           H
-ATOM     22  N   LYS A   4       9.206  21.532  23.211  1.00  0.00           N
-ATOM     23  CA  LYS A   4       9.863  22.178  22.082  1.00  0.00           C
-ATOM     24  C   LYS A   4      11.288  22.530  22.477  1.00  0.00           C
-ATOM     25  O   LYS A   4      11.504  23.213  23.482  1.00  0.00           O
-ATOM     26  CB  LYS A   4       9.103  23.436  21.645  1.00  0.00           C
-ATOM     27  CG  LYS A   4       9.729  24.159  20.463  1.00  0.00           C
-ATOM     28  CD  LYS A   4       8.928  25.393  20.084  1.00  0.00           C
-ATOM     29  CE  LYS A   4       9.556  26.119  18.906  1.00  0.00           C
-ATOM     30  NZ  LYS A   4       8.775  27.325  18.516  1.00  0.00           N1+
-ATOM      0  H   LYS A   4       9.467  21.814  23.981  1.00  0.00           H
-ATOM      0  HA  LYS A   4       9.872  21.567  21.329  1.00  0.00           H
-ATOM      0  HB2 LYS A   4       8.193  23.190  21.416  1.00  0.00           H
-ATOM      0  HB3 LYS A   4       9.051  24.048  22.396  1.00  0.00           H
-ATOM      0  HG2 LYS A   4      10.638  24.416  20.683  1.00  0.00           H
-ATOM      0  HG3 LYS A   4       9.780  23.558  19.703  1.00  0.00           H
-ATOM      0  HD2 LYS A   4       8.020  25.136  19.861  1.00  0.00           H
-ATOM      0  HD3 LYS A   4       8.874  25.992  20.845  1.00  0.00           H
-ATOM      0  HE2 LYS A   4      10.462  26.381  19.134  1.00  0.00           H
-ATOM      0  HE3 LYS A   4       9.617  25.515  18.149  1.00  0.00           H
-ATOM      0  HZ1 LYS A   4       9.170  27.724  17.826  1.00  0.00           H
-ATOM      0  HZ2 LYS A   4       7.950  27.083  18.286  1.00  0.00           H
-ATOM      0  HZ3 LYS A   4       8.738  27.890  19.203  1.00  0.00           H
-ATOM     31  N   ASP A   5      12.251  22.063  21.685  1.00  0.00           N
-ATOM     32  CA  ASP A   5      13.662  22.321  21.940  1.00  0.00           C
-ATOM     33  C   ASP A   5      14.368  22.572  20.617  1.00  0.00           C
-ATOM     34  O   ASP A   5      14.175  21.821  19.656  1.00  0.00           O
-ATOM     35  CB  ASP A   5      14.318  21.148  22.678  1.00  0.00           C
-ATOM     36  CG  ASP A   5      15.782  21.396  22.982  1.00  0.00           C
-ATOM     37  OD1 ASP A   5      16.154  22.567  23.210  1.00  0.00           O
-ATOM     38  OD2 ASP A   5      16.562  20.421  22.993  1.00  0.00           O1-
-ATOM      0  H   ASP A   5      12.102  21.587  20.984  1.00  0.00           H
-ATOM      0  HA  ASP A   5      13.739  23.104  22.508  1.00  0.00           H
-ATOM      0  HB2 ASP A   5      13.842  20.985  23.507  1.00  0.00           H
-ATOM      0  HB3 ASP A   5      14.234  20.345  22.141  1.00  0.00           H
-ATOM     39  N   GLU A   6      15.182  23.622  20.573  1.00  0.00           N
-ATOM     40  CA  GLU A   6      15.919  23.969  19.364  1.00  0.00           C
-ATOM     41  C   GLU A   6      17.320  24.468  19.701  1.00  0.00           C
-ATOM     42  O   GLU A   6      17.684  24.583  20.871  1.00  0.00           O
-ATOM     43  CB  GLU A   6      15.162  25.028  18.560  1.00  0.00           C
-ATOM     44  CG  GLU A   6      15.859  25.448  17.276  1.00  0.00           C
-ATOM     45  CD  GLU A   6      15.084  26.502  16.509  1.00  0.00           C
-ATOM     46  OE1 GLU A   6      13.994  26.896  16.975  1.00  0.00           O
-ATOM     47  OE2 GLU A   6      15.564  26.937  15.442  1.00  0.00           O1-
-ATOM      0  H   GLU A   6      15.322  24.149  21.238  1.00  0.00           H
-ATOM      0  HA  GLU A   6      16.003  23.167  18.825  1.00  0.00           H
-ATOM      0  HB2 GLU A   6      14.281  24.686  18.341  1.00  0.00           H
-ATOM      0  HB3 GLU A   6      15.032  25.811  19.117  1.00  0.00           H
-ATOM      0  HG2 GLU A   6      16.741  25.791  17.488  1.00  0.00           H
-ATOM      0  HG3 GLU A   6      15.985  24.670  16.711  1.00  0.00           H
+ATOM      1  N   ARG A   2       3.050  19.481  25.856  1.00  0.00           N
+ATOM      2  CA  ARG A   2       3.759  20.265  24.852  1.00  0.00           C
+ATOM      3  C   ARG A   2       5.253  20.304  25.155  1.00  0.00           C
+ATOM      4  O   ARG A   2       5.659  20.516  26.298  1.00  0.00           O
+ATOM      5  CB  ARG A   2       3.197  21.687  24.787  1.00  0.00           C
+ATOM      6  CG  ARG A   2       3.800  22.542  23.684  1.00  0.00           C
+ATOM      7  CD  ARG A   2       3.128  23.904  23.607  1.00  0.00           C
+ATOM      8  NE  ARG A   2       3.721  24.751  22.576  1.00  0.00           N
+ATOM      9  CZ  ARG A   2       3.311  25.983  22.290  1.00  0.00           C
+ATOM     10  NH1 ARG A   2       2.301  26.525  22.957  1.00  0.00           N1+
+ATOM     11  NH2 ARG A   2       3.914  26.676  21.334  1.00  0.00           N
+ATOM     12  HA  ARG A   2       3.638  19.852  23.983  1.00  0.00           H
+ATOM     13  HB2 ARG A   2       2.240  21.637  24.633  1.00  0.00           H
+ATOM     14  HB3 ARG A   2       3.370  22.129  25.633  1.00  0.00           H
+ATOM     15  HG2 ARG A   2       4.743  22.680  23.864  1.00  0.00           H
+ATOM     16  HG3 ARG A   2       3.682  22.096  22.831  1.00  0.00           H
+ATOM     17  HD2 ARG A   2       2.189  23.783  23.396  1.00  0.00           H
+ATOM     18  HD3 ARG A   2       3.224  24.355  24.460  1.00  0.00           H
+ATOM     19  HE  ARG A   2       4.380  24.432  22.124  1.00  0.00           H
+ATOM     20 HH11 ARG A   2       1.905  26.082  23.579  1.00  0.00           H
+ATOM     21 HH12 ARG A   2       2.041  27.322  22.767  1.00  0.00           H
+ATOM     22 HH21 ARG A   2       4.569  26.329  20.898  1.00  0.00           H
+ATOM     23 HH22 ARG A   2       3.649  27.473  21.149  1.00  0.00           H
+ATOM     24  N   HIS A   3       6.067  20.098  24.123  1.00  0.00           N
+ATOM     25  CA  HIS A   3       7.516  20.107  24.260  1.00  0.00           C
+ATOM     26  C   HIS A   3       8.133  20.732  23.018  1.00  0.00           C
+ATOM     27  O   HIS A   3       7.638  20.536  21.904  1.00  0.00           O
+ATOM     28  CB  HIS A   3       8.069  18.691  24.464  1.00  0.00           C
+ATOM     29  CG  HIS A   3       7.630  18.050  25.743  1.00  0.00           C
+ATOM     30  ND1 HIS A   3       8.154  18.399  26.969  1.00  0.00           N
+ATOM     31  CD2 HIS A   3       6.716  17.080  25.988  1.00  0.00           C
+ATOM     32  CE1 HIS A   3       7.582  17.674  27.913  1.00  0.00           C
+ATOM     33  NE2 HIS A   3       6.706  16.865  27.344  1.00  0.00           N
+ATOM     34  H   HIS A   3       5.797  19.948  23.320  1.00  0.00           H
+ATOM     35  HA  HIS A   3       7.764  20.646  25.027  1.00  0.00           H
+ATOM     36  HB2 HIS A   3       7.768  18.129  23.732  1.00  0.00           H
+ATOM     37  HB3 HIS A   3       9.038  18.731  24.470  1.00  0.00           H
+ATOM     38  HD2 HIS A   3       6.194  16.642  25.355  1.00  0.00           H
+ATOM     39  HE1 HIS A   3       7.765  17.723  28.824  1.00  0.00           H
+ATOM     40  N   LYS A   4       9.213  21.483  23.217  1.00  0.00           N
+ATOM     41  CA  LYS A   4       9.902  22.139  22.116  1.00  0.00           C
+ATOM     42  C   LYS A   4      11.340  22.409  22.530  1.00  0.00           C
+ATOM     43  O   LYS A   4      11.602  22.774  23.678  1.00  0.00           O
+ATOM     44  CB  LYS A   4       9.205  23.447  21.722  1.00  0.00           C
+ATOM     45  CG  LYS A   4       9.799  24.121  20.495  1.00  0.00           C
+ATOM     46  CD  LYS A   4       9.008  25.357  20.102  1.00  0.00           C
+ATOM     47  CE  LYS A   4       9.606  26.035  18.880  1.00  0.00           C
+ATOM     48  NZ  LYS A   4       8.842  27.250  18.484  1.00  0.00           N1+
+ATOM     49  H   LYS A   4       9.568  21.628  23.987  1.00  0.00           H
+ATOM     50  HA  LYS A   4       9.909  21.551  21.344  1.00  0.00           H
+ATOM     51  HB2 LYS A   4       8.272  23.258  21.533  1.00  0.00           H
+ATOM     52  HB3 LYS A   4       9.270  24.070  22.462  1.00  0.00           H
+ATOM     53  HG2 LYS A   4      10.710  24.392  20.688  1.00  0.00           H
+ATOM     54  HG3 LYS A   4       9.784  23.500  19.750  1.00  0.00           H
+ATOM     55  HD2 LYS A   4       8.096  25.101  19.892  1.00  0.00           H
+ATOM     56  HD3 LYS A   4       9.018  25.991  20.837  1.00  0.00           H
+ATOM     57  HE2 LYS A   4      10.517  26.302  19.078  1.00  0.00           H
+ATOM     58  HE3 LYS A   4       9.594  25.415  18.135  1.00  0.00           H
+ATOM     59  HZ1 LYS A   4       9.215  27.624  17.768  1.00  0.00           H
+ATOM     60  HZ2 LYS A   4       8.001  27.031  18.292  1.00  0.00           H
+ATOM     61  HZ3 LYS A   4       8.842  27.840  19.151  1.00  0.00           H
+ATOM     62  N   ASP A   5      12.263  22.226  21.588  1.00  0.00           N
+ATOM     63  CA  ASP A   5      13.678  22.447  21.845  1.00  0.00           C
+ATOM     64  C   ASP A   5      14.374  22.763  20.531  1.00  0.00           C
+ATOM     65  O   ASP A   5      14.035  22.197  19.489  1.00  0.00           O
+ATOM     66  CB  ASP A   5      14.324  21.224  22.508  1.00  0.00           C
+ATOM     67  CG  ASP A   5      15.729  21.504  23.005  1.00  0.00           C
+ATOM     68  OD1 ASP A   5      16.213  22.640  22.819  1.00  0.00           O
+ATOM     69  OD2 ASP A   5      16.350  20.587  23.582  1.00  0.00           O1-
+ATOM     70  H   ASP A   5      12.091  21.972  20.785  1.00  0.00           H
+ATOM     71  HA  ASP A   5      13.783  23.208  22.438  1.00  0.00           H
+ATOM     72  HB2 ASP A   5      13.785  20.954  23.267  1.00  0.00           H
+ATOM     73  HB3 ASP A   5      14.372  20.503  21.861  1.00  0.00           H
+ATOM     74  N   GLU A   6      15.346  23.668  20.589  1.00  0.00           N
+ATOM     75  CA  GLU A   6      16.096  24.064  19.403  1.00  0.00           C
+ATOM     76  C   GLU A   6      17.433  24.684  19.794  1.00  0.00           C
+ATOM     77  O   GLU A   6      17.829  24.647  20.959  1.00  0.00           O
+ATOM     78  CB  GLU A   6      15.283  25.051  18.561  1.00  0.00           C
+ATOM     79  CG  GLU A   6      15.942  25.433  17.245  1.00  0.00           C
+ATOM     80  CD  GLU A   6      15.076  26.352  16.405  1.00  0.00           C
+ATOM     81  OE1 GLU A   6      13.969  26.708  16.861  1.00  0.00           O
+ATOM     82  OE2 GLU A   6      15.503  26.718  15.290  1.00  0.00           O1-
+ATOM     83  H   GLU A   6      15.592  24.070  21.309  1.00  0.00           H
+ATOM     84  HA  GLU A   6      16.273  23.279  18.862  1.00  0.00           H
+ATOM     85  HB2 GLU A   6      14.424  24.651  18.356  1.00  0.00           H
+ATOM     86  HB3 GLU A   6      15.153  25.864  19.073  1.00  0.00           H
+ATOM     87  HG2 GLU A   6      16.776  25.893  17.428  1.00  0.00           H
+ATOM     88  HG3 GLU A   6      16.112  24.628  16.730  1.00  0.00           H
 TER
-ATOM     48  N   SER B   2      10.369   5.142  24.245  1.00  0.00           N
-ATOM     49  CA  SER B   2      11.201   6.333  24.366  1.00  0.00           C
-ATOM     50  C   SER B   2      12.675   5.962  24.488  1.00  0.00           C
-ATOM     51  O   SER B   2      13.050   5.137  25.320  1.00  0.00           O
-ATOM     52  CB  SER B   2      10.767   7.171  25.571  1.00  0.00           C
-ATOM     53  OG  SER B   2      10.895   6.435  26.775  1.00  0.00           O
-ATOM      0  HA  SER B   2      11.085   6.860  23.560  1.00  0.00           H
-ATOM      0  HB2 SER B   2      11.307   7.975  25.621  1.00  0.00           H
-ATOM      0  HB3 SER B   2       9.846   7.454  25.457  1.00  0.00           H
-ATOM      0  HG  SER B   2      10.808   5.615  26.613  1.00  0.00           H
-ATOM     54  N   THR B   3      13.508   6.579  23.652  1.00  0.00           N
-ATOM     55  CA  THR B   3      14.944   6.326  23.652  1.00  0.00           C
-ATOM     56  C   THR B   3      15.671   7.653  23.507  1.00  0.00           C
-ATOM     57  O   THR B   3      15.545   8.321  22.477  1.00  0.00           O
-ATOM     58  CB  THR B   3      15.343   5.371  22.522  1.00  0.00           C
-ATOM     59  OG1 THR B   3      14.664   4.120  22.689  1.00  0.00           O
-ATOM     60  CG2 THR B   3      16.846   5.132  22.533  1.00  0.00           C
-ATOM      0  H   THR B   3      13.254   7.157  23.068  1.00  0.00           H
-ATOM      0  HA  THR B   3      15.191   5.901  24.488  1.00  0.00           H
-ATOM      0  HB  THR B   3      15.094   5.771  21.674  1.00  0.00           H
-ATOM      0  HG1 THR B   3      14.882   3.598  22.068  1.00  0.00           H
-ATOM      0 HG21 THR B   3      17.083   4.526  21.813  1.00  0.00           H
-ATOM      0 HG22 THR B   3      17.309   5.976  22.411  1.00  0.00           H
-ATOM      0 HG23 THR B   3      17.106   4.741  23.382  1.00  0.00           H
-ATOM     61  N   ASN B   4      16.427   8.031  24.535  1.00  0.00           N
-ATOM     62  CA  ASN B   4      17.183   9.276  24.541  1.00  0.00           C
-ATOM     63  C   ASN B   4      18.633   8.977  24.886  1.00  0.00           C
-ATOM     64  O   ASN B   4      18.908   8.244  25.841  1.00  0.00           O
-ATOM     65  CB  ASN B   4      16.594  10.278  25.541  1.00  0.00           C
-ATOM     66  CG  ASN B   4      15.152  10.631  25.231  1.00  0.00           C
-ATOM     67  OD1 ASN B   4      14.224  10.007  25.745  1.00  0.00           O
-ATOM     68  ND2 ASN B   4      14.957  11.638  24.387  1.00  0.00           N
-ATOM      0  H   ASN B   4      16.515   7.567  25.254  1.00  0.00           H
-ATOM      0  HA  ASN B   4      17.132   9.677  23.659  1.00  0.00           H
-ATOM      0  HB2 ASN B   4      16.648   9.907  26.436  1.00  0.00           H
-ATOM      0  HB3 ASN B   4      17.130  11.087  25.537  1.00  0.00           H
-ATOM      0 HD21 ASN B   4      14.157  11.877  24.180  1.00  0.00           H
-ATOM      0 HD22 ASN B   4      15.631  12.051  24.047  1.00  0.00           H
-ATOM     69  N   GLN B   5      19.552   9.543  24.112  1.00  0.00           N
-ATOM     70  CA  GLN B   5      20.978   9.337  24.339  1.00  0.00           C
-ATOM     71  C   GLN B   5      21.781  10.569  23.935  1.00  0.00           C
-ATOM     72  O   GLN B   5      21.698  11.030  22.797  1.00  0.00           O
-ATOM     73  CB  GLN B   5      21.472   8.111  23.569  1.00  0.00           C
-ATOM     74  CG  GLN B   5      22.953   7.818  23.751  1.00  0.00           C
-ATOM     75  CD  GLN B   5      23.407   6.598  22.974  1.00  0.00           C
-ATOM     76  OE1 GLN B   5      24.582   6.232  23.001  1.00  0.00           O
-ATOM     77  NE2 GLN B   5      22.474   5.961  22.275  1.00  0.00           N
-ATOM      0  H   GLN B   5      19.369  10.054  23.445  1.00  0.00           H
-ATOM      0  HA  GLN B   5      21.110   9.185  25.288  1.00  0.00           H
-ATOM      0  HB2 GLN B   5      20.962   7.336  23.852  1.00  0.00           H
-ATOM      0  HB3 GLN B   5      21.291   8.240  22.625  1.00  0.00           H
-ATOM      0  HG2 GLN B   5      23.469   8.589  23.466  1.00  0.00           H
-ATOM      0  HG3 GLN B   5      23.139   7.684  24.694  1.00  0.00           H
-ATOM      0 HE21 GLN B   5      21.663   6.247  22.280  1.00  0.00           H
-ATOM      0 HE22 GLN B   5      22.681   5.263  21.817  1.00  0.00           H
+ATOM     89  N   SER B   2      10.846   4.462  24.534  1.00  0.00           N
+ATOM     90  CA  SER B   2      11.518   5.747  24.391  1.00  0.00           C
+ATOM     91  C   SER B   2      13.032   5.575  24.457  1.00  0.00           C
+ATOM     92  O   SER B   2      13.532   4.595  25.009  1.00  0.00           O
+ATOM     93  CB  SER B   2      11.052   6.719  25.477  1.00  0.00           C
+ATOM     94  OG  SER B   2      11.455   6.279  26.762  1.00  0.00           O
+ATOM     95  HA  SER B   2      11.295   6.129  23.528  1.00  0.00           H
+ATOM     96  HB2 SER B   2      11.440   7.591  25.307  1.00  0.00           H
+ATOM     97  HB3 SER B   2      10.084   6.777  25.453  1.00  0.00           H
+ATOM     98  HG  SER B   2      11.125   5.523  26.924  1.00  0.00           H
+ATOM     99  N   THR B   3      13.758   6.536  23.889  1.00  0.00           N
+ATOM    100  CA  THR B   3      15.212   6.498  23.880  1.00  0.00           C
+ATOM    101  C   THR B   3      15.741   7.920  23.778  1.00  0.00           C
+ATOM    102  O   THR B   3      15.212   8.733  23.015  1.00  0.00           O
+ATOM    103  CB  THR B   3      15.746   5.652  22.717  1.00  0.00           C
+ATOM    104  OG1 THR B   3      15.230   4.319  22.813  1.00  0.00           O
+ATOM    105  CG2 THR B   3      17.268   5.606  22.740  1.00  0.00           C
+ATOM    106  H   THR B   3      13.425   7.226  23.499  1.00  0.00           H
+ATOM    107  HA  THR B   3      15.530   6.111  24.711  1.00  0.00           H
+ATOM    108  HB  THR B   3      15.465   6.046  21.876  1.00  0.00           H
+ATOM    109  HG1 THR B   3      15.519   3.852  22.178  1.00  0.00           H
+ATOM    110 HG21 THR B   3      17.587   4.933  22.118  1.00  0.00           H
+ATOM    111 HG22 THR B   3      17.633   6.467  22.484  1.00  0.00           H
+ATOM    112 HG23 THR B   3      17.580   5.381  23.631  1.00  0.00           H
+ATOM    113  N   ASN B   4      16.785   8.212  24.552  1.00  0.00           N
+ATOM    114  CA  ASN B   4      17.402   9.533  24.564  1.00  0.00           C
+ATOM    115  C   ASN B   4      18.904   9.364  24.725  1.00  0.00           C
+ATOM    116  O   ASN B   4      19.359   8.747  25.692  1.00  0.00           O
+ATOM    117  CB  ASN B   4      16.835  10.400  25.693  1.00  0.00           C
+ATOM    118  CG  ASN B   4      15.357  10.690  25.518  1.00  0.00           C
+ATOM    119  OD1 ASN B   4     ...
 
[truncated message content]

Revision: 25553
          http://sourceforge.net/p/cctbx/code/25553
Author:   bkpoon
Date:     2016-10-05 22:01:45 +0000 (Wed, 05 Oct 2016)
Log Message:
-----------
Bug fix for miller array parsing function where helpful message about how to select a label is not shown

Modified Paths:
--------------
    trunk/iotbx/reflection_file_utils.py

Modified: trunk/iotbx/reflection_file_utils.py
===================================================================
--- trunk/iotbx/reflection_file_utils.py	2016-10-05 21:47:39 UTC (rev 25552)
+++ trunk/iotbx/reflection_file_utils.py	2016-10-05 22:01:45 UTC (rev 25553)
@@ -866,22 +866,21 @@
       print >> log, "    ", ma.info().label_string()
       suitable_arrays.append(ma)
       suitable_labels.append(ma.info().label_string())
+  print >> log
   if(  len(suitable_arrays) == 0): raise Sorry("No suitable arrays.")
   elif(len(suitable_arrays) == 1): result = suitable_arrays[0]
   elif(len(suitable_arrays) >  1):
-    m="Multiple choices available."
-    if(label is None): raise Sorry(m)
+    if(label is None):
+      msg='''Multiple choices available. No map coefficients array selected.
+
+  See choices listed above and use "label=" to select one.
+  Example: label="2FOFCWT,PH2FOFCWT"'''
+      raise Sorry(msg)
     else:
       for ma in miller_arrays:
         if(get_flag(ma=ma) and (ma.info().label_string() == label)):
           print >> log, "  Selected:", ma.info().label_string()
           result = ma
-  if(result is None):
-    msg="""
-No map coefficients array selected. See choices listed above and use 'label='
-to select one. Example: label="2FOFCWT,PH2FOFCWT"
-"""
-    raise Sorry(msg)
   return result
 
 class process_raw_data (object) :

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

Revision: 25552
          http://sourceforge.net/p/cctbx/code/25552
Author:   olegsobolev
Date:     2016-10-05 21:47:39 +0000 (Wed, 05 Oct 2016)
Log Message:
-----------
remove duplicated import, commented testing output.

Modified Paths:
--------------
    trunk/mmtbx/utils/__init__.py

Modified: trunk/mmtbx/utils/__init__.py
===================================================================
--- trunk/mmtbx/utils/__init__.py	2016-10-05 15:58:17 UTC (rev 25551)
+++ trunk/mmtbx/utils/__init__.py	2016-10-05 21:47:39 UTC (rev 25552)
@@ -55,7 +55,6 @@
 from iotbx.cns.miller_array import crystal_symmetry_as_cns_comments
 from iotbx.file_reader import any_file
 from mmtbx.rotamer.rotamer_eval import RotamerEval
-from cctbx import maptbx
 
 import boost.python
 utils_ext = boost.python.import_ext("mmtbx_utils_ext")
@@ -2690,6 +2689,7 @@
             if evaluation == "Allowed" and accept_allowed:
               exclude = True
           if not exclude:
+            # print "  Fixing rotamer outlier", residue.id_str()
             rotamer_iterator = lockit.get_rotamer_iterator(
               mon_lib_srv         = mon_lib_srv,
               residue             = residue,

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.