modelutils¶
This module provides utilities to build Cython models based on Python models automatically.
Most model developers do not need to be aware of the features
implemented in module modelutils
, except that they need to
initialise class Cythonizer
within the main modules of their base
and application models (see, for example, the source code of base
model hland
and application model hland_v1
).
However, when implementing models with functionalities not envisaged
so far, problems might arise. Please contact the HydPy
developer team then, preferably by opening an issue on GitHub.
Potentially, problems could occur when defining parameters or sequences
with larger dimensionality than anticipated. The following
example shows the Cython code lines for the get_point_states()
method of class ELSModel
, used for deriving the test
model. By
now, we did only implement 0-dimensional and 1-dimensional sequences
requiring this method. After hackishly changing the dimensionality of
sequences S
, we still seem to get plausible results, but
these are untested in model applications:
>>> from hydpy.models.test import cythonizer
>>> pyxwriter = cythonizer.pyxwriter
>>> pyxwriter.get_point_states
. get_point_states
cpdef inline void get_point_states(self) nogil:
cdef int idx0
self.sequences.states.s = self.sequences.states._s_points[self.numvars.idx_stage]
for idx0 in range(self.sequences.states._sv_length):
self.sequences.states.sv[idx0] = self.sequences.states._sv_points[self.numvars.idx_stage][idx0]
>>> pyxwriter.model.sequences.states.s.NDIM = 2
>>> pyxwriter.get_point_states
. get_point_states
cpdef inline void get_point_states(self) nogil:
cdef int idx0, idx1
for idx0 in range(self.sequences.states._s_length0):
for idx1 in range(self.sequences.states._s_length1):
self.sequences.states.s[idx0, idx1] = self.sequences.states._s_points[self.numvars.idx_stage][idx0, idx1]
for idx0 in range(self.sequences.states._sv_length):
self.sequences.states.sv[idx0] = self.sequences.states._sv_points[self.numvars.idx_stage][idx0]
>>> pyxwriter.model.sequences.states.s.NDIM = 3
>>> pyxwriter.get_point_states
Traceback (most recent call last):
...
NotImplementedError: NDIM of sequence `s` is higher than expected.
The following examples show the results for some methods which are also
related to numerical integration but deal with FluxSequence
objects.
We start with the method integrate_fluxes()
:
>>> pyxwriter.integrate_fluxes
. integrate_fluxes
cpdef inline void integrate_fluxes(self) nogil:
cdef int jdx, idx0
self.sequences.fluxes.q = 0.
for jdx in range(self.numvars.idx_method):
self.sequences.fluxes.q = self.sequences.fluxes.q +self.numvars.dt * self.numconsts.a_coefs[self.numvars.idx_method-1, self.numvars.idx_stage, jdx]*self.sequences.fluxes._q_points[jdx]
for idx0 in range(self.sequences.fluxes._qv_length):
self.sequences.fluxes.qv[idx0] = 0.
for jdx in range(self.numvars.idx_method):
self.sequences.fluxes.qv[idx0] = self.sequences.fluxes.qv[idx0] + self.numvars.dt * self.numconsts.a_coefs[self.numvars.idx_method-1, self.numvars.idx_stage, jdx]*self.sequences.fluxes._qv_points[jdx, idx0]
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 2
>>> pyxwriter.integrate_fluxes
. integrate_fluxes
cpdef inline void integrate_fluxes(self) nogil:
cdef int jdx, idx0, idx1
for idx0 in range(self.sequences.fluxes._q_length0):
for idx1 in range(self.sequences.fluxes._q_length1):
self.sequences.fluxes.q[idx0, idx1] = 0.
for jdx in range(self.numvars.idx_method):
self.sequences.fluxes.q[idx0, idx1] = self.sequences.fluxes.q[idx0, idx1] + self.numvars.dt * self.numconsts.a_coefs[self.numvars.idx_method-1, self.numvars.idx_stage, jdx]*self.sequences.fluxes._q_points[jdx, idx0, idx1]
for idx0 in range(self.sequences.fluxes._qv_length):
self.sequences.fluxes.qv[idx0] = 0.
for jdx in range(self.numvars.idx_method):
self.sequences.fluxes.qv[idx0] = self.sequences.fluxes.qv[idx0] + self.numvars.dt * self.numconsts.a_coefs[self.numvars.idx_method-1, self.numvars.idx_stage, jdx]*self.sequences.fluxes._qv_points[jdx, idx0]
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 3
>>> pyxwriter.integrate_fluxes
Traceback (most recent call last):
...
NotImplementedError: NDIM of sequence `q` is higher than expected.
Method reset_sum_fluxes()
:
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 0
>>> pyxwriter.reset_sum_fluxes
. reset_sum_fluxes
cpdef inline void reset_sum_fluxes(self) nogil:
cdef int idx0
self.sequences.fluxes._q_sum = 0.
for idx0 in range(self.sequences.fluxes._qv_length):
self.sequences.fluxes._qv_sum[idx0] = 0.
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 2
>>> pyxwriter.reset_sum_fluxes
. reset_sum_fluxes
cpdef inline void reset_sum_fluxes(self) nogil:
cdef int idx0, idx1
for idx0 in range(self.sequences.fluxes._q_length0):
for idx1 in range(self.sequences.fluxes._q_length1):
self.sequences.fluxes._q_sum[idx0, idx1] = 0.
for idx0 in range(self.sequences.fluxes._qv_length):
self.sequences.fluxes._qv_sum[idx0] = 0.
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 3
>>> pyxwriter.reset_sum_fluxes
Traceback (most recent call last):
...
NotImplementedError: NDIM of sequence `q` is higher than expected.
Method addup_fluxes()
:
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 0
>>> pyxwriter.addup_fluxes
. addup_fluxes
cpdef inline void addup_fluxes(self) nogil:
cdef int idx0
self.sequences.fluxes._q_sum = self.sequences.fluxes._q_sum + self.sequences.fluxes.q
for idx0 in range(self.sequences.fluxes._qv_length):
self.sequences.fluxes._qv_sum[idx0] = self.sequences.fluxes._qv_sum[idx0] + self.sequences.fluxes.qv[idx0]
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 2
>>> pyxwriter.addup_fluxes
. addup_fluxes
cpdef inline void addup_fluxes(self) nogil:
cdef int idx0, idx1
for idx0 in range(self.sequences.fluxes._q_length0):
for idx1 in range(self.sequences.fluxes._q_length1):
self.sequences.fluxes._q_sum[idx0, idx1] = self.sequences.fluxes._q_sum[idx0, idx1] + self.sequences.fluxes.q[idx0, idx1]
for idx0 in range(self.sequences.fluxes._qv_length):
self.sequences.fluxes._qv_sum[idx0] = self.sequences.fluxes._qv_sum[idx0] + self.sequences.fluxes.qv[idx0]
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 3
>>> pyxwriter.addup_fluxes
Traceback (most recent call last):
...
NotImplementedError: NDIM of sequence `q` is higher than expected.
Method calculate_error()
:
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 0
>>> pyxwriter.calculate_error
. calculate_error
cpdef inline void calculate_error(self) nogil:
cdef int idx0
cdef double abserror
self.numvars.abserror = 0.
if self.numvars.use_relerror:
self.numvars.relerror = 0.
else:
self.numvars.relerror = inf
abserror = fabs(self.sequences.fluxes._q_results[self.numvars.idx_method]-self.sequences.fluxes._q_results[self.numvars.idx_method-1])
self.numvars.abserror = max(self.numvars.abserror, abserror)
if self.numvars.use_relerror:
if self.sequences.fluxes._q_results[self.numvars.idx_method] == 0.:
self.numvars.relerror = inf
else:
self.numvars.relerror = max(self.numvars.relerror, fabs(abserror/self.sequences.fluxes._q_results[self.numvars.idx_method]))
for idx0 in range(self.sequences.fluxes._qv_length):
abserror = fabs(self.sequences.fluxes._qv_results[self.numvars.idx_method, idx0]-self.sequences.fluxes._qv_results[self.numvars.idx_method-1, idx0])
self.numvars.abserror = max(self.numvars.abserror, abserror)
if self.numvars.use_relerror:
if self.sequences.fluxes._qv_results[self.numvars.idx_method, idx0] == 0.:
self.numvars.relerror = inf
else:
self.numvars.relerror = max(self.numvars.relerror, fabs(abserror/self.sequences.fluxes._qv_results[self.numvars.idx_method, idx0]))
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 2
>>> pyxwriter.calculate_error
. calculate_error
cpdef inline void calculate_error(self) nogil:
cdef int idx0, idx1
cdef double abserror
self.numvars.abserror = 0.
if self.numvars.use_relerror:
self.numvars.relerror = 0.
else:
self.numvars.relerror = inf
for idx0 in range(self.sequences.fluxes._q_length0):
for idx1 in range(self.sequences.fluxes._q_length1):
abserror = fabs(self.sequences.fluxes._q_results[self.numvars.idx_method, idx0, idx1]-self.sequences.fluxes._q_results[self.numvars.idx_method-1, idx0, idx1])
self.numvars.abserror = max(self.numvars.abserror, abserror)
if self.numvars.use_relerror:
if self.sequences.fluxes._q_results[self.numvars.idx_method, idx0, idx1] == 0.:
self.numvars.relerror = inf
else:
self.numvars.relerror = max(self.numvars.relerror, fabs(abserror/self.sequences.fluxes._q_results[self.numvars.idx_method, idx0, idx1]))
for idx0 in range(self.sequences.fluxes._qv_length):
abserror = fabs(self.sequences.fluxes._qv_results[self.numvars.idx_method, idx0]-self.sequences.fluxes._qv_results[self.numvars.idx_method-1, idx0])
self.numvars.abserror = max(self.numvars.abserror, abserror)
if self.numvars.use_relerror:
if self.sequences.fluxes._qv_results[self.numvars.idx_method, idx0] == 0.:
self.numvars.relerror = inf
else:
self.numvars.relerror = max(self.numvars.relerror, fabs(abserror/self.sequences.fluxes._qv_results[self.numvars.idx_method, idx0]))
>>> pyxwriter.model.sequences.fluxes.q.NDIM = 3
>>> pyxwriter.calculate_error
Traceback (most recent call last):
...
NotImplementedError: NDIM of sequence `q` is higher than expected.
Module modelutils
implements the following members:
get_dllextension()
Return the DLL file extension for the current operating system.
Lines
Handles code lines for .pyx file.
get_methodheader()
Returns the Cython method header for methods without arguments except self.
decorate_method()
The decorated method returns aLines
object including a method header. However, theLines
object is empty if the respective model does not implement a method with the same name as the wrapped method.
Cythonizer
Handles the writing, compiling and initialisation of Cython models.
PyxWriter
Translates the source code of Python models into Cython source code.
FuncConverter
Helper class for classPyxWriter
that analyses Python functions and provides the required Cython code via propertypyxlines
.
exp()
Cython wrapper for theexp()
function of modulenumpy
applied on a singlefloat
object.
log()
Cython wrapper for thelog()
function of modulenumpy
applied on a singlefloat
object.
fabs()
Cython wrapper for theexp()
function of modulemath
applied on a singlefloat
object.
sin()
Cython wrapper for thesin()
function of modulenumpy
applied on a singlefloat
object.
cos()
Cython wrapper for thecos()
function of modulenumpy
applied on a singlefloat
object.
tan()
Cython wrapper for thetan()
function of modulenumpy
applied on a singlefloat
object.
asin()
Cython wrapper for thearcsin()
function of modulenumpy
applied on a singlefloat
object.
acos()
Cython wrapper for thearccos()
function of modulenumpy
applied on a singlefloat
object.
atan()
Cython wrapper for thearctan()
function of modulenumpy
applied on a singlefloat
object.
isnan()
Cython wrapper for theisnan()
function of modulenumpy
applied on a singlefloat
object.
isinf()
Cython wrapper for theisinf()
function of modulenumpy
applied on a singlefloat
object.
-
hydpy.cythons.modelutils.
get_dllextension
() → str[source]¶ Return the DLL file extension for the current operating system.
The returned value depends on the response of function
system()
of moduleplatform
.get_dllextension()
returns .pyd ifsystem()
returns the string “windows” and .so for all other strings:>>> from hydpy.cythons.modelutils import get_dllextension >>> import platform >>> from unittest import mock >>> with mock.patch.object( ... platform, "system", side_effect=lambda: "Windows") as mocked: ... get_dllextension() '.pyd' >>> with mock.patch.object( ... platform, "system", side_effect=lambda: "Linux") as mocked: ... get_dllextension() '.so'
-
hydpy.cythons.modelutils.
TYPE2STR
: Dict[Optional[Type[Any]], str] = {<class 'bool'>: 'bint', <class 'int'>: 'numpy.int64_t', <class 'hydpy.core.parametertools.IntConstant'>: 'numpy.int64_t', <class 'float'>: 'double', <class 'str'>: 'str', None: 'void', <class 'hydpy.core.typingtools.Vector'>: 'double[:]', hydpy.core.typingtools.Vector[float]: 'double[:]'}¶ Maps Python types to Cython compatible type declarations.
The Cython type belonging to Python’s
int
is selected to agree with numpy’s default integer type on the current platform/system.
-
hydpy.cythons.modelutils.
CHECKABLE_TYPES
: Tuple[Type[Any], …] = (<class 'bool'>, <class 'int'>, <class 'hydpy.core.parametertools.IntConstant'>, <class 'float'>, <class 'str'>, <class 'hydpy.core.typingtools.Vector'>)¶ “Real types” of
TYPE2STR
allowed as second arguments of functionisinstance()
.
-
hydpy.cythons.modelutils.
get_methodheader
(methodname: str, nogil: bool = False, idxarg: bool = False) → str[source]¶ Returns the Cython method header for methods without arguments except self.
Note the influence of the configuration flag FASTCYTHON:
>>> from hydpy.cythons.modelutils import get_methodheader >>> from hydpy import config >>> config.FASTCYTHON = False >>> print(get_methodheader(methodname="test", nogil=True, idxarg=False)) cpdef inline void test(self): >>> config.FASTCYTHON = True >>> print(get_methodheader(methodname="test", nogil=True, idxarg=True)) cpdef inline void test(self, int idx) nogil:
-
hydpy.cythons.modelutils.
decorate_method
(wrapped: Callable) → property[source]¶ The decorated method returns a
Lines
object including a method header. However, theLines
object is empty if the respective model does not implement a method with the same name as the wrapped method.
-
class
hydpy.cythons.modelutils.
Cythonizer
[source]¶ Bases:
object
Handles the writing, compiling and initialisation of Cython models.
-
Model
: Type[hydpy.core.modeltools.Model]¶
-
Parameters
: Type[hydpy.core.parametertools.Parameters]¶
-
Sequences
: Type[hydpy.core.sequencetools.Sequences]¶
-
tester
: hydpy.core.testtools.Tester¶
-
finalise
() → None[source]¶ Test and cythonize the relevant model eventually.
Method
finalise()
might call methodcythonize()
and methodperform_tests()
depending on the actual values of the optionsautocompile
,usecython
, andskipdoctests
as well the value currently returned by propertyoutdated
. To explain and test the considerable amount of relevant combinations, we make use of Python’sunittest
mock library.First, we import the
Cythonizer
instance responsible for application modelhland_v1
, the classesCythonizer
andTester
, modulepub
, and theunittest
mock library:>>> from hydpy.models.hland_v1 import cythonizer >>> from hydpy.cythons.modelutils import Cythonizer >>> from hydpy.core.testtools import Tester >>> from hydpy import pub >>> from unittest import mock
Second, we memorise the relevant settings to restore them later:
>>> autocompile = pub.options.autocompile >>> skipdoctests = pub.options.skipdoctests >>> usecython = pub.options.usecython >>> outdated = Cythonizer.outdated
Third, we define a test function mocking methods
cythonize()
andperform_tests()
, printing when the mocks are called and providing information on the current value of optionusecython
:>>> def test(): ... sc = lambda: print( ... f"calling method `cythonize` " ... f"(usecython={bool(pub.options.usecython)})") ... se = lambda: print( ... f"calling method `perform_tests` " ... f"(usecython={bool(pub.options.usecython)})") ... with mock.patch.object( ... Cythonizer, "cythonize", side_effect=sc) as mc,\ ... mock.patch.object( ... Tester, "perform_tests", side_effect=se) as mt: ... cythonizer.finalise()
With either option
autocompile
or propertyoutdated
beingFalse
, nothing happens:>>> pub.options.autocompile = False >>> Cythonizer.outdated = True >>> test()
>>> pub.options.autocompile = True >>> Cythonizer.outdated = False >>> test()
Option
usecython
enables/disables the actual cythonization and optionskipdoctests
enables/disables the testing of the Python model and, if available, of the Cython model:>>> Cythonizer.outdated = True >>> pub.options.usecython = False >>> pub.options.skipdoctests = True >>> test()
>>> pub.options.skipdoctests = False >>> test() calling method `perform_tests` (usecython=False)
>>> pub.options.usecython = True >>> pub.options.skipdoctests = True >>> test() calling method `cythonize` (usecython=True)
>>> pub.options.skipdoctests = False >>> test() calling method `perform_tests` (usecython=False) calling method `cythonize` (usecython=False) calling method `perform_tests` (usecython=True)
>>> pub.options.autocompile = autocompile >>> Cythonizer.outdated = outdated >>> pub.options.skipdoctests = skipdoctests
-
cythonize
() → None[source]¶ Translate Python source code of the relevant model first into Cython and then into C, compile it, and move the resulting dll file to the autogen subfolder of subpackage cythons.
-
property
pyname
¶ Name of the original Python module or package.
>>> from hydpy.models.hland import cythonizer >>> cythonizer.pyname 'hland' >>> from hydpy.models.hland_v1 import cythonizer >>> cythonizer.pyname 'hland_v1'
-
property
cyname
¶ Name of the compiled module.
>>> from hydpy.models.hland import cythonizer >>> cythonizer.cyname 'c_hland' >>> from hydpy.models.hland_v1 import cythonizer >>> cythonizer.cyname 'c_hland_v1'
-
property
cydirpath
¶ The absolute path of the directory containing the compiled modules.
>>> from hydpy.models.hland import cythonizer >>> from hydpy import repr_ >>> repr_(cythonizer.cydirpath) '.../hydpy/cythons/autogen' >>> import os >>> os.path.exists(cythonizer.cydirpath) True
-
property
cymodule
¶ The compiled module.
Property
cymodule
returns the relevant DLL module:>>> from hydpy.models.hland_v1 import cythonizer >>> from hydpy.cythons.autogen import c_hland_v1 >>> c_hland_v1 is cythonizer.cymodule True
However, if this module is missing for some reasons, it tries to create the module first and returns it afterwards. For demonstration purposes, we define a wrong
cyname
:>>> from hydpy.cythons.modelutils import Cythonizer >>> cyname = Cythonizer.cyname >>> Cythonizer.cyname = "wrong" >>> cythonizer._cymodule = None >>> from unittest import mock >>> with mock.patch.object(Cythonizer, "cythonize") as mock: ... cythonizer.cymodule Traceback (most recent call last): ... ModuleNotFoundError: No module named 'hydpy.cythons.autogen.wrong' >>> mock.call_args_list [call()]
>>> Cythonizer.cyname = cyname
-
property
pyxfilepath
¶ The absolute path of the compiled module.
>>> from hydpy.models.hland_v1 import cythonizer >>> from hydpy import repr_ >>> repr_(cythonizer.pyxfilepath) '.../hydpy/cythons/autogen/c_hland_v1.pyx' >>> import os >>> os.path.exists(cythonizer.pyxfilepath) True
-
property
dllfilepath
¶ The absolute path of the compiled module.
>>> from hydpy.models.hland_v1 import cythonizer >>> from hydpy import repr_ >>> repr_(cythonizer.dllfilepath) '.../hydpy/cythons/autogen/c_hland_v1...' >>> import os >>> os.path.exists(cythonizer.dllfilepath) True
-
property
buildpath
¶ The absolute path for temporarily build files.
>>> from hydpy.models.hland_v1 import cythonizer >>> from hydpy import repr_ >>> repr_(cythonizer.buildpath) '.../hydpy/cythons/autogen/_build'
-
property
pyxwriter
¶ A new
PyxWriter
instance.>>> from hydpy.models.hland_v1 import cythonizer >>> pyxwriter = cythonizer.pyxwriter >>> from hydpy import classname >>> classname(pyxwriter) 'PyxWriter' >>> cythonizer.pyxwriter is pyxwriter False
-
property
pysourcefiles
¶ All relevant source files of the actual model.
We consider source files to be relevant if they are part of the HydPy package and if they define ancestors of the classes of the considered model.
For the base model
hland
, all relevant modules seem to be covered:>>> from hydpy.models.hland import cythonizer >>> import os, pprint >>> pprint.pprint([fn.split(os.path.sep)[-1] for fn in ... sorted(cythonizer.pysourcefiles)]) ['masktools.py', 'modeltools.py', 'parametertools.py', 'sequencetools.py', 'testtools.py', 'variabletools.py', 'modelutils.py', '__init__.py', 'hland_masks.py', 'hland_model.py']
However, this is not the case for application model
hland_v1
, where the base model files are missing. Hence, relevant changes in its base model might not be detected, resulting in an outdated application model. This issue is relevant for developers only, but we should fix it someday:>>> from hydpy.models.hland_v1 import cythonizer >>> import os, pprint >>> pprint.pprint([fn.split(os.path.sep)[-1] for fn in ... sorted(cythonizer.pysourcefiles)]) ['masktools.py', 'modeltools.py', 'parametertools.py', 'sequencetools.py', 'testtools.py', 'variabletools.py', 'modelutils.py', 'hland_v1.py']
-
property
outdated
¶ True/False flag indicating whether a
Cythonizer
object should renew its Cython model or not.With option
forcecompiling
beingTrue
, propertyoutdated
also returnTrue
under all circumstances:>>> from hydpy.models.hland_v1 import cythonizer >>> from hydpy import pub >>> forcecompiling = pub.options.forcecompiling >>> pub.options.forcecompiling = True >>> cythonizer.outdated True
With option
forcecompiling
beingFalse
, propertyoutdated
generally returnFalse
if HydPy is a site-package (under the assumption the user does not modify his site-package files and for reasons of efficiency due to skipping the following tests):>>> pub.options.forcecompiling = False >>> from unittest import mock >>> with mock.patch("hydpy.__path__", ["folder/somename-packages/hydpy"]): ... cythonizer.outdated False >>> with mock.patch("hydpy.__path__", ["folder/pkgs/hydpy"]): ... cythonizer.outdated False
When working with a “local” HydPy package (that is not part of the site-packages directory) property
outdated
returnsTrue
if the required DLL file is not available at all:>>> with mock.patch("hydpy.__path__", ["folder/local_dir/hydpy"]): ... with mock.patch.object( ... type(cythonizer), "dllfilepath", ... new_callable=mock.PropertyMock) as dllfilepath: ... dllfilepath.return_value = "missing" ... cythonizer.outdated True
If the DLL file is available, property
outdated
returnsTrue
orFalse
depending on the timestamp of the DLL file itself and the timestamp of the newest file returned by propertypysourcefiles
:>>> from hydpy import TestIO >>> with TestIO(): ... with open("new.txt", "w"): ... pass ... with mock.patch("hydpy.__path__", ["folder/local_dir/hydpy"]): ... with mock.patch.object( ... type(cythonizer), "dllfilepath", ... new_callable=mock.PropertyMock) as mocked: ... mocked.return_value = "new.txt" ... cythonizer.outdated ... with mock.patch.object( ... type(cythonizer), "pysourcefiles", ... new_callable=mock.PropertyMock) as mocked: ... mocked.return_value = ["new.txt"] ... cythonizer.outdated False True
>>> pub.options.forcecompiling = forcecompiling
-
move_dll
() → None[source]¶ Try to find the DLL file created my method
compile_()
and to move it into the autogen folder of the cythons subpackage.Usually, one does not need to apply the
move_dll()
method directly. However, if you are a model developer, you might see one of the following error messages from time to time:>>> from hydpy.models.hland_v1 import cythonizer >>> cythonizer.move_dll() Traceback (most recent call last): ... OSError: After trying to cythonize model `hland_v1`, the resulting file `c_hland_v1...` could not be found in directory `.../hydpy/cythons/autogen/_build` nor any of its subdirectories. The distutil report should tell whether the file has been stored somewhere else, is named somehow else, or could not be build at all.
>>> import os >>> from unittest import mock >>> from hydpy import TestIO >>> with TestIO(): ... with mock.patch.object( ... type(cythonizer), "buildpath", new_callable=mock.PropertyMock ... ) as mocked_buildpath: ... mocked_buildpath.return_value = "_build" ... os.makedirs("_build/subdir", exist_ok=True) ... filepath = f"_build/subdir/c_hland_v1{get_dllextension()}" ... with open(filepath, "w"): ... pass ... with mock.patch( ... "shutil.move", ... side_effect=PermissionError("Denied!")): ... cythonizer.move_dll() Traceback (most recent call last): ... PermissionError: After trying to cythonize module `hland_v1`, when trying to move the final cython module `c_hland_v1...` from directory `_build` to directory `.../hydpy/cythons/autogen`, the following error occurred: Denied! A likely error cause is that the cython module `c_hland_v1...` does already exist in this directory and is currently blocked by another Python process. Maybe it helps to close all Python processes and restart the cythonization afterwards.
-
-
class
hydpy.cythons.modelutils.
PyxWriter
(cythonizer: hydpy.cythons.modelutils.Cythonizer, model: hydpy.core.modeltools.Model, pyxpath: str)[source]¶ Bases:
object
Translates the source code of Python models into Cython source code.
Method
PyxWriter
serves as a master method, which triggers the complete writing process. The other properties and methods supply the required code lines. Their names are selected to match the names of the original Python models as close as possible.-
cythonizer
: hydpy.cythons.modelutils.Cythonizer¶
-
model
: hydpy.core.modeltools.Model¶
-
property
cythondistutilsoptions
¶ Cython and Distutils option lines.
Use the configuration options “FASTCYTHON” and “PROFILECYTHON” to configure the cythonization processes as follows:
>>> from hydpy.cythons.modelutils import PyxWriter >>> pyxwriter = PyxWriter(None, None, None) >>> pyxwriter.cythondistutilsoptions #!python # cython: language_level=3 # cython: boundscheck=False # cython: wraparound=False # cython: initializedcheck=False # cython: cdivision=True
>>> from hydpy import config >>> config.FASTCYTHON = False >>> config.PROFILECYTHON = True >>> pyxwriter.cythondistutilsoptions #!python # cython: language_level=3 # cython: boundscheck=True # cython: wraparound=True # cython: initializedcheck=True # cython: cdivision=False # cython: linetrace=True # distutils: define_macros=CYTHON_TRACE=1 # distutils: define_macros=CYTHON_TRACE_NOGIL=1
>>> config.FASTCYTHON = True >>> config.PROFILECYTHON = False
-
property
cimports
¶ Import command lines.
-
property
constants
¶ Constants declaration lines.
-
property
parameters
¶ Parameter declaration lines.
-
property
sequences
¶ Sequence declaration lines.
-
static
iosequence
(seq: hydpy.core.sequencetools.IOSequence) → List[str][source]¶ Declaration lines for the given
IOSequence
object.
-
static
open_files
(subseqs: hydpy.core.sequencetools.IOSequences) → List[str][source]¶ Open file statements.
-
static
close_files
(subseqs: hydpy.core.sequencetools.IOSequences) → List[str][source]¶ Close file statements.
-
static
load_data
(subseqs: hydpy.core.sequencetools.IOSequences) → List[str][source]¶ Load data statements.
-
static
save_data
(subseqs: hydpy.core.sequencetools.IOSequences) → List[str][source]¶ Save data statements.
-
set_pointer
(subseqs: Union[hydpy.core.sequencetools.InputSequences, hydpy.core.sequencetools.OutputSequences, hydpy.core.sequencetools.LinkSequences]) → List[str][source]¶ Set pointer statements for all input, output, and link sequences.
-
static
set_pointer0d
(subseqs: hydpy.core.sequencetools.LinkSequences) → List[str][source]¶ Set pointer statements for 0-dimensional link sequences.
-
static
get_value
(subseqs: hydpy.core.sequencetools.LinkSequences) → List[str][source]¶ Get value statements for link sequences.
-
static
set_value
(subseqs: hydpy.core.sequencetools.LinkSequences) → List[str][source]¶ Set value statements for link sequences.
-
static
alloc
(subseqs: hydpy.core.sequencetools.LinkSequences) → List[str][source]¶ Allocate memory statements for 1-dimensional link sequences.
-
static
dealloc
(subseqs: hydpy.core.sequencetools.LinkSequences) → List[str][source]¶ Deallocate memory statements for 1-dimensional link sequences.
-
static
set_pointer1d
(subseqs: hydpy.core.sequencetools.LinkSequences) → List[str][source]¶ Set_pointer statements for 1-dimensional link sequences.
-
static
set_pointerinput
(subseqs: hydpy.core.sequencetools.InputSequences) → List[str][source]¶ Set pointer statements for input sequences.
-
set_pointeroutput
(subseqs: hydpy.core.sequencetools.OutputSequences) → List[str][source]¶ Set pointer statements for output sequences.
-
property
numericalparameters
¶ Numeric parameter declaration lines.
-
property
submodels
¶ Submodel declaration lines.
-
property
modeldeclarations
¶ The attribute declarations of the model class.
-
property
modelstandardfunctions
¶ The standard functions of the model class.
-
property
modelnumericfunctions
¶ Numerical integration functions of the model class.
-
property
simulate
¶ Simulation statements.
-
property
iofunctions
¶ Input/output functions of the model class.
The result of property
iofunctions
depends on the availability of different types of sequences. So far, the models implemented in HydPy do not reflect all possible combinations, which is why we modify thehland_v1
application model in the following examples:>>> from hydpy.models.hland_v1 import cythonizer >>> pyxwriter = cythonizer.pyxwriter >>> pyxwriter.iofunctions . open_files . close_files . load_data . save_data cpdef inline void open_files(self): self.sequences.inputs.open_files(self.idx_sim) self.sequences.fluxes.open_files(self.idx_sim) self.sequences.states.open_files(self.idx_sim) cpdef inline void close_files(self): self.sequences.inputs.close_files() self.sequences.fluxes.close_files() self.sequences.states.close_files() cpdef inline void load_data(self) nogil: self.sequences.inputs.load_data(self.idx_sim) cpdef inline void save_data(self, int idx) nogil: self.sequences.inputs.save_data(self.idx_sim) self.sequences.fluxes.save_data(self.idx_sim) self.sequences.states.save_data(self.idx_sim)
>>> pyxwriter.model.sequences.fluxes = None >>> pyxwriter.model.sequences.states = None >>> pyxwriter.iofunctions . open_files . close_files . load_data . save_data cpdef inline void open_files(self): self.sequences.inputs.open_files(self.idx_sim) cpdef inline void close_files(self): self.sequences.inputs.close_files() cpdef inline void load_data(self) nogil: self.sequences.inputs.load_data(self.idx_sim) cpdef inline void save_data(self, int idx) nogil: self.sequences.inputs.save_data(self.idx_sim)
>>> pyxwriter.model.sequences.inputs = None >>> pyxwriter.iofunctions
-
property
new2old
¶ Old states to new states statements.
-
property
update_receivers
¶ Lines of the model method with the same name.
-
property
update_inlets
¶ Lines of the model method with the same name.
-
run
(model: hydpy.core.modeltools.AdHocModel) → List[str][source]¶ Return the lines of the model method with the same name.
-
property
update_outlets
¶ Lines of the model method with the same name.
-
property
update_senders
¶ Lines of the model method with the same name.
-
property
update_outputs_model
¶ Lines of the model method with the same name (except the _model suffix).
-
update_outputs
(subseqs: hydpy.core.sequencetools.OutputSequences) → List[str][source]¶ Lines of the subsequences method with the same name.
-
calculate_single_terms
(model: hydpy.core.modeltools.SolverModel) → List[str][source]¶ Return the lines of the model method with the same name.
-
calculate_full_terms
(model: hydpy.core.modeltools.SolverModel) → List[str][source]¶ Return the lines of the model method with the same name.
-
property
listofmodeluserfunctions
¶ User functions of the model class.
-
property
modeluserfunctions
¶ Model-specific functions.
-
property
solve
¶ Lines of the model method with the same name.
-
property
get_point_states
¶ Lines of model method get_point_states.
-
property
set_point_states
¶ Lines of model method set_point_states.
-
property
set_result_states
¶ Lines of model method set_result_states.
-
property
get_sum_fluxes
¶ Lines of model method get_sum_fluxes.
-
property
set_point_fluxes
¶ Lines of model method set_point_fluxes.
-
property
set_result_fluxes
¶ Lines of model method set_result_fluxes.
-
property
integrate_fluxes
¶ Lines of model method integrate_fluxes.
-
property
reset_sum_fluxes
¶ Lines of model method reset_sum_fluxes.
-
property
addup_fluxes
¶ Lines of model method addup_fluxes.
-
property
calculate_error
¶ Lines of model method calculate_error.
-
property
extrapolate_error
¶ Extrapolate error statements.
-
write_stubfile
()[source]¶ Write a stub file for the actual base or application model.
At the moment, HydPy creates model objects quite dynamically. In many regards, this comes with lots of conveniences. However, there two critical drawbacks compared to more static approaches: some amount of additional initialisation time and, more important, much opaqueness for code inspection tools. In this context, we experiment with “stub files” at the moment. These could either contain typing information only or define statically predefined model classes. The following example uses method
write_stubfile()
to write a (far from perfect) prototype stub file for base modelhland
:>>> from hydpy.models.hland import * >>> cythonizer.pyxwriter.write_stubfile()
This is the path to the written file:
>>> import os >>> import hydpy >>> filepath = os.path.join(hydpy.__path__[0], "hland.py") >>> os.path.exists(filepath) True
However, it’s just an experimental prototype, so we better remove it:
>>> os.remove(filepath) >>> os.path.exists(filepath) False
-
-
class
hydpy.cythons.modelutils.
FuncConverter
(model: hydpy.core.modeltools.Model, funcname: str, func: Callable)[source]¶ Bases:
object
Helper class for class
PyxWriter
that analyses Python functions and provides the required Cython code via propertypyxlines
.-
func
: Callable¶
-
model
: hydpy.core.modeltools.Model¶
-
property
argnames
¶ The argument names of the current function.
>>> from hydpy.cythons.modelutils import FuncConverter >>> from hydpy import prepare_model, pub >>> with pub.options.usecython(False): ... model = prepare_model("hland_v1") >>> FuncConverter(model, None, model.calc_tc_v1).argnames ['model']
-
property
varnames
¶ The variable names of the current function.
>>> from hydpy.cythons.modelutils import FuncConverter >>> from hydpy import prepare_model, pub >>> with pub.options.usecython(False): ... model = prepare_model("hland_v1") >>> FuncConverter(model, None, model.calc_tc_v1).varnames ('self', 'con', 'inp', 'flu', 'k')
-
property
locnames
¶ The variable names of the handled function except for the argument names.
>>> from hydpy.cythons.modelutils import FuncConverter >>> from hydpy import prepare_model, pub >>> with pub.options.usecython(False): ... model = prepare_model("hland_v1") >>> FuncConverter(model, None, model.calc_tc_v1).locnames ['self', 'con', 'inp', 'flu', 'k']
-
property
subgroupnames
¶ The complete names of the subgroups relevant for the current function.
>>> from hydpy.cythons.modelutils import FuncConverter >>> from hydpy import prepare_model, pub >>> with pub.options.usecython(False): ... model = prepare_model("hland_v1") >>> FuncConverter(model, None, model.calc_tc_v1).subgroupnames ['parameters.control', 'sequences.inputs', 'sequences.fluxes']
-
property
subgroupshortcuts
¶ The abbreviated names of the subgroups relevant for the current function.
>>> from hydpy.cythons.modelutils import FuncConverter >>> from hydpy import prepare_model, pub >>> with pub.options.usecython(False): ... model = prepare_model("hland_v1") >>> FuncConverter(model, None, model.calc_tc_v1).subgroupshortcuts ['con', 'inp', 'flu']
-
property
untypedvarnames
¶ The names of the untyped variables used in the current function.
>>> from hydpy.cythons.modelutils import FuncConverter >>> from hydpy import prepare_model, pub >>> with pub.options.usecython(False): ... model = prepare_model("hland_v1") >>> FuncConverter(model, None, model.calc_tc_v1).untypedvarnames ['k']
-
property
untypedarguments
¶ The names of the untyped arguments used by the current function.
>>> from hydpy.cythons.modelutils import FuncConverter >>> from hydpy import prepare_model, pub >>> with pub.options.usecython(False): ... model = prepare_model("hland_v1") >>> FuncConverter(model, None, model.calc_tc_v1).untypedarguments []
-
property
untypedinternalvarnames
¶ The names of the untyped variables used in the current function except for those of the arguments.
>>> from hydpy.cythons.modelutils import FuncConverter >>> from hydpy import prepare_model, pub >>> with pub.options.usecython(False): ... model = prepare_model("hland_v1") >>> FuncConverter(model, None, model.calc_tc_v1).untypedinternalvarnames ['k']
-
property
cleanlines
¶ The leaned code lines of the current function.
- The implemented cleanups:
eventually, remove method version
remove all docstrings
remove all comments
remove all empty lines
remove line bracks within brackets
remove the phrase modelutils
remove all lines containing the phrase fastaccess
replace all shortcuts with complete reference names
replace “model.” with “self.”
-
static
remove_linebreaks_within_equations
(code: str) → str[source]¶ Remove line breaks within equations.
The following example is not an exhaustive test but shows how the method works in principle:
>>> code = "asdf = \\\n(a\n+b)" >>> from hydpy.cythons.modelutils import FuncConverter >>> FuncConverter.remove_linebreaks_within_equations(code) 'asdf = (a+b)'
-
static
remove_imath_operators
(lines: List[str])[source]¶ Remove mathematical expressions that require Pythons global interpreter locking mechanism.
The following example is not an exhaustive test but shows how the method works in principle:
>>> lines = [" x += 1*1"] >>> from hydpy.cythons.modelutils import FuncConverter >>> FuncConverter.remove_imath_operators(lines) >>> lines [' x = x + (1*1)']
-
property
pyxlines
¶ Cython code lines of the current function.
- Assumptions:
The function shall be a method.
The method shall be inlined.
Annotations specify all argument and return types.
Local variables are generally of type int but of type double when their name starts with d_.
We import some classes and prepare a pure-Python instance of application model
hland_v1
:>>> from types import MethodType >>> from hydpy.core.modeltools import Method, Model >>> from hydpy.core.typingtools import Vector >>> from hydpy.cythons.modelutils import FuncConverter >>> from hydpy import prepare_model, pub >>> with pub.options.usecython(False): ... model = prepare_model("hland_v1")
First, we show an example on a standard method without additional arguments and returning nothing but requiring two local variables:
>>> class Calc_Test_V1(Method): ... @staticmethod ... def __call__(model: Model) -> None: ... con = model.parameters.control.fastaccess ... flu = model.sequences.fluxes.fastaccess ... inp = model.sequences.inputs.fastaccess ... for k in range(con.nmbzones): ... d_pc = con.kg[k]*inp.p[k] ... flu.pc[k] = d_pc >>> model.calc_test_v1 = MethodType(Calc_Test_V1.__call__, model) >>> FuncConverter(model, "calc_test_v1", model.calc_test_v1).pyxlines cpdef inline void calc_test_v1(self) nogil: cdef double d_pc cdef int k for k in range(self.parameters.control.nmbzones): d_pc = self.parameters.control.kg[k]*self.sequences.inputs.p[k] self.sequences.fluxes.pc[k] = d_pc
The second example shows that float and Vector annotations translate into double and double[:] types, respectively:
>>> class Calc_Test_V2(Method): ... @staticmethod ... def __call__( ... model: Model, value: float, values: Vector) -> float: ... con = model.parameters.control.fastaccess ... return con.kg[0]*value*values[1] >>> model.calc_test_v2 = MethodType(Calc_Test_V2.__call__, model) >>> FuncConverter(model, "calc_test_v2", model.calc_test_v2).pyxlines cpdef inline double calc_test_v2(self, double value, double[:] values) nogil: return self.parameters.control.kg[0]*value*values[1]
-
-
hydpy.cythons.modelutils.
exp
(double: float) → float[source]¶ Cython wrapper for the
exp()
function of modulenumpy
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import exp >>> from unittest import mock >>> with mock.patch("numpy.exp") as func: ... _ = exp(123.4) >>> func.call_args call(123.4)
-
hydpy.cythons.modelutils.
log
(double: float) → float[source]¶ Cython wrapper for the
log()
function of modulenumpy
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import log >>> from unittest import mock >>> with mock.patch("numpy.log") as func: ... _ = log(123.4) >>> func.call_args call(123.4)
-
hydpy.cythons.modelutils.
fabs
(double: float) → float[source]¶ Cython wrapper for the
exp()
function of modulemath
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import fabs >>> from unittest import mock >>> with mock.patch("math.fabs") as func: ... _ = fabs(123.4) >>> func.call_args call(123.4)
-
hydpy.cythons.modelutils.
sin
(double: float) → float[source]¶ Cython wrapper for the
sin()
function of modulenumpy
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import sin >>> from unittest import mock >>> with mock.patch("numpy.sin") as func: ... _ = sin(123.4) >>> func.call_args call(123.4)
-
hydpy.cythons.modelutils.
cos
(double: float) → float[source]¶ Cython wrapper for the
cos()
function of modulenumpy
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import cos >>> from unittest import mock >>> with mock.patch("numpy.cos") as func: ... _ = cos(123.4) >>> func.call_args call(123.4)
-
hydpy.cythons.modelutils.
tan
(double: float) → float[source]¶ Cython wrapper for the
tan()
function of modulenumpy
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import tan >>> from unittest import mock >>> with mock.patch("numpy.tan") as func: ... _ = tan(123.4) >>> func.call_args call(123.4)
-
hydpy.cythons.modelutils.
asin
(double: float) → float[source]¶ Cython wrapper for the
arcsin()
function of modulenumpy
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import asin >>> from unittest import mock >>> with mock.patch("numpy.arcsin") as func: ... _ = asin(123.4) >>> func.call_args call(123.4)
-
hydpy.cythons.modelutils.
acos
(double: float) → float[source]¶ Cython wrapper for the
arccos()
function of modulenumpy
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import acos >>> from unittest import mock >>> with mock.patch("numpy.arccos") as func: ... _ = acos(123.4) >>> func.call_args call(123.4)
-
hydpy.cythons.modelutils.
atan
(double: float) → float[source]¶ Cython wrapper for the
arctan()
function of modulenumpy
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import atan >>> from unittest import mock >>> with mock.patch("numpy.arctan") as func: ... _ = atan(123.4) >>> func.call_args call(123.4)
-
hydpy.cythons.modelutils.
isnan
(double: float) → float[source]¶ Cython wrapper for the
isnan()
function of modulenumpy
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import isnan >>> from unittest import mock >>> with mock.patch("numpy.isnan") as func: ... _ = isnan(123.4) >>> func.call_args call(123.4)
-
hydpy.cythons.modelutils.
isinf
(double: float) → float[source]¶ Cython wrapper for the
isinf()
function of modulenumpy
applied on a singlefloat
object.>>> from hydpy.cythons.modelutils import isnan >>> from unittest import mock >>> with mock.patch("numpy.isinf") as func: ... _ = isinf(123.4) >>> func.call_args call(123.4)