gt
stringclasses 1
value | context
stringlengths 2.49k
119k
|
|---|---|
""" Model definition functions and weight loading.
"""
from __future__ import print_function, division
from keras.models import Model, Sequential
from keras.layers.merge import concatenate
from keras.layers import Input, Bidirectional, Embedding, Dense, Dropout, SpatialDropout1D, LSTM, Activation
from keras.regularizers import L1L2
from attlayer import AttentionWeightedAverage
from global_variables import NB_TOKENS, NB_EMOJI_CLASSES
import numpy as np
from copy import deepcopy
from os.path import exists
import h5py
def deepmoji_feature_encoding(maxlen, weight_path, return_attention=False):
""" Loads the pretrained DeepMoji model for extracting features
from the penultimate feature layer. In this way, it transforms
the text into its emotional encoding.
# Arguments:
maxlen: Maximum length of a sentence (given in tokens).
weight_path: Path to model weights to be loaded.
return_attention: If true, output will be weight of each input token
used for the prediction
# Returns:
Pretrained model for encoding text into feature vectors.
"""
model = deepmoji_architecture(nb_classes=None, nb_tokens=NB_TOKENS,
maxlen=maxlen, feature_output=True,
return_attention=return_attention)
load_specific_weights(model, weight_path, exclude_names=['softmax'])
return model
def deepmoji_emojis(maxlen, weight_path, return_attention=False):
""" Loads the pretrained DeepMoji model for extracting features
from the penultimate feature layer. In this way, it transforms
the text into its emotional encoding.
# Arguments:
maxlen: Maximum length of a sentence (given in tokens).
weight_path: Path to model weights to be loaded.
return_attention: If true, output will be weight of each input token
used for the prediction
# Returns:
Pretrained model for encoding text into feature vectors.
"""
model = deepmoji_architecture(nb_classes=NB_EMOJI_CLASSES,
nb_tokens=NB_TOKENS, maxlen=maxlen,
return_attention=return_attention)
model.load_weights(weight_path, by_name=False)
return model
def deepmoji_transfer(nb_classes, maxlen, weight_path=None, extend_embedding=0,
embed_dropout_rate=0.25, final_dropout_rate=0.5,
embed_l2=1E-6):
""" Loads the pretrained DeepMoji model for finetuning/transfer learning.
Does not load weights for the softmax layer.
Note that if you are planning to use class average F1 for evaluation,
nb_classes should be set to 2 instead of the actual number of classes
in the dataset, since binary classification will be performed on each
class individually.
Note that for the 'new' method, weight_path should be left as None.
# Arguments:
nb_classes: Number of classes in the dataset.
maxlen: Maximum length of a sentence (given in tokens).
weight_path: Path to model weights to be loaded.
extend_embedding: Number of tokens that have been added to the
vocabulary on top of NB_TOKENS. If this number is larger than 0,
the embedding layer's dimensions are adjusted accordingly, with the
additional weights being set to random values.
embed_dropout_rate: Dropout rate for the embedding layer.
final_dropout_rate: Dropout rate for the final Softmax layer.
embed_l2: L2 regularization for the embedding layerl.
# Returns:
Model with the given parameters.
"""
model = deepmoji_architecture(nb_classes=nb_classes,
nb_tokens=NB_TOKENS + extend_embedding,
maxlen=maxlen, embed_dropout_rate=embed_dropout_rate,
final_dropout_rate=final_dropout_rate, embed_l2=embed_l2)
if weight_path is not None:
load_specific_weights(model, weight_path,
exclude_names=['softmax'],
extend_embedding=extend_embedding)
return model
def deepmoji_architecture(nb_classes, nb_tokens, maxlen, feature_output=False, embed_dropout_rate=0, final_dropout_rate=0, embed_l2=1E-6, return_attention=False):
"""
Returns the DeepMoji architecture uninitialized and
without using the pretrained model weights.
# Arguments:
nb_classes: Number of classes in the dataset.
nb_tokens: Number of tokens in the dataset (i.e. vocabulary size).
maxlen: Maximum length of a token.
feature_output: If True the model returns the penultimate
feature vector rather than Softmax probabilities
(defaults to False).
embed_dropout_rate: Dropout rate for the embedding layer.
final_dropout_rate: Dropout rate for the final Softmax layer.
embed_l2: L2 regularization for the embedding layerl.
# Returns:
Model with the given parameters.
"""
# define embedding layer that turns word tokens into vectors
# an activation function is used to bound the values of the embedding
model_input = Input(shape=(maxlen,), dtype='int32')
embed_reg = L1L2(l2=embed_l2) if embed_l2 != 0 else None
embed = Embedding(input_dim=nb_tokens,
output_dim=256,
mask_zero=True,
input_length=maxlen,
embeddings_regularizer=embed_reg,
name='embedding')
x = embed(model_input)
x = Activation('tanh')(x)
# entire embedding channels are dropped out instead of the
# normal Keras embedding dropout, which drops all channels for entire words
# many of the datasets contain so few words that losing one or more words can alter the emotions completely
if embed_dropout_rate != 0:
embed_drop = SpatialDropout1D(embed_dropout_rate, name='embed_drop')
x = embed_drop(x)
# skip-connection from embedding to output eases gradient-flow and allows access to lower-level features
# ordering of the way the merge is done is important for consistency with the pretrained model
lstm_0_output = Bidirectional(LSTM(512, return_sequences=True), name="bi_lstm_0")(x)
lstm_1_output = Bidirectional(LSTM(512, return_sequences=True), name="bi_lstm_1")(lstm_0_output)
x = concatenate([lstm_1_output, lstm_0_output, x])
# if return_attention is True in AttentionWeightedAverage, an additional tensor
# representing the weight at each timestep is returned
weights = None
x = AttentionWeightedAverage(name='attlayer', return_attention=return_attention)(x)
if return_attention:
x, weights = x
if not feature_output:
# output class probabilities
if final_dropout_rate != 0:
x = Dropout(final_dropout_rate)(x)
if nb_classes > 2:
outputs = [Dense(nb_classes, activation='softmax', name='softmax')(x)]
else:
outputs = [Dense(1, activation='sigmoid', name='softmax')(x)]
else:
# output penultimate feature vector
outputs = [x]
if return_attention:
# add the attention weights to the outputs if required
outputs.append(weights)
return Model(inputs=[model_input], outputs=outputs, name="DeepMoji")
def load_specific_weights(model, weight_path, exclude_names=[], extend_embedding=0, verbose=True):
""" Loads model weights from the given file path, excluding any
given layers.
# Arguments:
model: Model whose weights should be loaded.
weight_path: Path to file containing model weights.
exclude_names: List of layer names whose weights should not be loaded.
extend_embedding: Number of new words being added to vocabulary.
verbose: Verbosity flag.
# Raises:
ValueError if the file at weight_path does not exist.
"""
if not exists(weight_path):
raise ValueError('ERROR (load_weights): The weights file at {} does '
'not exist. Refer to the README for instructions.'
.format(weight_path))
if extend_embedding and 'embedding' in exclude_names:
raise ValueError('ERROR (load_weights): Cannot extend a vocabulary '
'without loading the embedding weights.')
# Copy only weights from the temporary model that are wanted
# for the specific task (e.g. the Softmax is often ignored)
layer_weights = get_weights_from_hdf5(weight_path)
for i, w in enumerate(layer_weights):
l_name = w[0]
weight_names = w[1]
weight_values = w[2]
if l_name in exclude_names:
if verbose:
print('Ignoring weights for {}'.format(l_name))
continue
try:
model_l = model.get_layer(name=l_name)
except ValueError:
raise ValueError("Weights had layer {},".format(l_name) +
" but could not find this layer in model.")
if verbose:
print('Loading weights for {}'.format(l_name))
# extend embedding layer to allow new randomly initialized words
# if requested. Otherwise, just load the weights for the layer.
if type(model_l) is Embedding and extend_embedding > 0:
comb_weights = append_to_embedding(weight_values,
model_l.get_weights())
model_l.set_weights(comb_weights)
if verbose:
print('Extended vocabulary for embedding layer ' +
'from {} to {} tokens.'.format(
NB_TOKENS, NB_TOKENS + extend_embedding))
else:
model_l.set_weights(weight_values)
def append_to_embedding(pretrain_weights, random_init_weights):
""" Uses pretrained weights for the tokens already in the vocabulary.
Remaining weights will be left with the random initialization. """
pretrain_weights = deepcopy(pretrain_weights)
if type(pretrain_weights) == list:
pretrain_weights = pretrain_weights[0]
if type(random_init_weights) == list:
random_init_weights = random_init_weights[0]
nb_old_tokens = np.shape(pretrain_weights)[0]
random_init_weights[:nb_old_tokens] = pretrain_weights
# must be returned as a list to be properly inserted into Keras model
return [random_init_weights]
def get_weights_from_hdf5(filepath):
""" Loads the weights from a saved Keras model into numpy arrays.
The weights are saved using Keras 2.0 so we don't need all the
conversion functionality for handling old weights.
"""
with h5py.File(filepath, mode='r') as f:
layer_names = [n.decode('utf8') for n in f.attrs['layer_names']]
layer_weights = []
for k, l_name in enumerate(layer_names):
g = f[l_name]
weight_names = [n.decode('utf8') for n in g.attrs['weight_names']]
weight_values = [g[weight_name][:] for weight_name in weight_names]
if len(weight_values):
layer_weights.append([l_name, weight_names, weight_values])
return layer_weights
|
|
#
# This is Seisflows
#
# See LICENCE file
#
###############################################################################
# Import system modules
import sys
# Import Numpy and Obspy
import numpy as np
import obspy
# Local imports
from seisflows.tools import msg, unix
from seisflows.tools.tools import exists, getset
from seisflows.config import ParameterError
from seisflows.plugins import adjoint, misfit, readers, writers
from seisflows.tools import signal
PAR = sys.modules['seisflows_parameters']
PATH = sys.modules['seisflows_paths']
class base(object):
""" Data preprocessing class
Provides data processing functions for seismic traces, with options for
data misfit, filtering, normalization and muting
"""
def check(self):
""" Checks parameters and paths
"""
# used for inversion
if 'MISFIT' not in PAR:
setattr(PAR, 'MISFIT', None)
# used for migration
if 'BACKPROJECT' not in PAR:
setattr(PAR, 'BACKPROJECT', None)
# data file format
if 'FORMAT' not in PAR:
raise ParameterError(PAR, 'FORMAT')
# data normalization option
if 'NORMALIZE' not in PAR:
setattr(PAR, 'NORMALIZE', None)
# data muting option
if 'MUTE' not in PAR:
setattr(PAR, 'MUTE', None)
# data filtering option
if 'FILTER' not in PAR:
setattr(PAR, 'FILTER', None)
# assertions
if PAR.FORMAT not in dir(readers):
print msg.ReaderError
raise ParameterError()
if PAR.FORMAT not in dir(writers):
print msg.WriterError
raise ParameterError()
self.check_filter()
self.check_mute()
self.check_normalize()
def setup(self):
""" Sets up data preprocessing machinery
"""
# define misfit function and adjoint trace generator
if PAR.MISFIT:
self.misfit = getattr(misfit, PAR.MISFIT)
self.adjoint = getattr(adjoint, PAR.MISFIT)
elif PAR.BACKPROJECT:
self.adjoint = getattr(adjoint, PAR.BACKPROJECT)
# define seismic data reader and writer
self.reader = getattr(readers, PAR.FORMAT)
self.writer = getattr(writers, PAR.FORMAT)
def prepare_eval_grad(self, path='.'):
"""
Prepares solver for gradient evaluation by writing residuals and
adjoint traces
:input path: directory containing observed and synthetic seismic data
"""
solver = sys.modules['seisflows_solver']
for filename in solver.data_filenames:
obs = self.reader(path+'/'+'traces/obs', filename)
syn = self.reader(path+'/'+'traces/syn', filename)
# process observations
obs = self.apply_filter(obs)
obs = self.apply_mute(obs)
obs = self.apply_normalize(obs)
# process synthetics
syn = self.apply_filter(syn)
syn = self.apply_mute(syn)
syn = self.apply_normalize(syn)
if PAR.MISFIT:
self.write_residuals(path, syn, obs)
self.write_adjoint_traces(path+'/'+'traces/adj', syn, obs,
filename)
def write_residuals(self, path, syn, obs):
"""
Computes residuals
:input path: location "adjoint traces" will be written
:input syn: obspy Stream object containing synthetic data
:input obs: obspy Stream object containing observed data
"""
nt, dt, _ = self.get_time_scheme(syn)
nn, _ = self.get_network_size(syn)
residuals = []
for ii in range(nn):
residuals.append(self.misfit(syn[ii].data, obs[ii].data, nt, dt))
filename = path+'/'+'residuals'
if exists(filename):
residuals.extend(list(np.loadtxt(filename)))
np.savetxt(filename, residuals)
def sum_residuals(self, files):
"""
Sums squares of residuals
:input files: list of single-column text files containing residuals
:output total_misfit: sum of squares of residuals
"""
total_misfit = 0.
for filename in files:
total_misfit += np.sum(np.loadtxt(filename)**2.)
return total_misfit
def write_adjoint_traces(self, path, syn, obs, channel):
"""
Writes "adjoint traces" required for gradient computation
:input path: location "adjoint traces" will be written
:input syn: obspy Stream object containing synthetic data
:input obs: obspy Stream object containing observed data
:input channel: channel or component code used by writer
"""
nt, dt, _ = self.get_time_scheme(syn)
nn, _ = self.get_network_size(syn)
adj = syn
for ii in range(nn):
adj[ii].data = self.adjoint(syn[ii].data, obs[ii].data, nt, dt)
self.writer(adj, path, channel)
# Signal processing
def apply_filter(self, traces):
if not PAR.FILTER:
return traces
elif PAR.FILTER == 'Bandpass':
for tr in traces:
tr.detrend('demean')
tr.detrend('linear')
tr.taper(0.05, type='hann')
tr.filter('bandpass',
zerophase=True,
freqmin=PAR.FREQMIN,
freqmax=PAR.FREQMAX)
elif PAR.FILTER == 'Lowpass':
for tr in traces:
tr.detrend('demean')
tr.detrend('linear')
tr.taper(0.05, type='hann')
tr.filter('lowpass',
zerophase=True,
freq=PAR.FREQ)
elif PAR.FILTER == 'Highpass':
for tr in traces:
tr.detrend('demean')
tr.detrend('linear')
tr.taper(0.05, type='hann')
tr.filter('highpass',
zerophase=True,
freq=PAR.FREQ)
else:
raise ParameterError()
return traces
def apply_mute(self, traces):
if not PAR.MUTE:
return traces
if 'MuteEarlyArrivals' in PAR.MUTE:
traces = signal.mute_early_arrivals(traces,
PAR.MUTE_EARLY_ARRIVALS_SLOPE, # (units: time/distance)
PAR.MUTE_EARLY_ARRIVALS_CONST, # (units: time)
self.get_time_scheme(traces),
self.get_source_coords(traces),
self.get_receiver_coords(traces))
if 'MuteLateArrivals' in PAR.MUTE:
traces = signal.mute_late_arrivals(traces,
PAR.MUTE_LATE_ARRIVALS_SLOPE, # (units: time/distance)
PAR.MUTE_LATE_ARRIVALS_CONST, # (units: time)
self.get_time_scheme(traces),
self.get_source_coords(traces),
self.get_receiver_coords(traces))
if 'MuteShortOffsets' in PAR.MUTE:
traces = signal.mute_short_offsets(traces,
PAR.MUTE_SHORT_OFFSETS_DIST,
self.get_source_coords(traces),
self.get_receiver_coords(traces))
if 'MuteLongOffsets' in PAR.MUTE:
traces = signal.mute_long_offsets(traces,
PAR.MUTE_LONG_OFFSETS_DIST,
self.get_source_coords(traces),
self.get_receiver_coords(traces))
return traces
def apply_normalize(self, traces):
if not PAR.NORMALIZE:
return traces
if 'NormalizeEventsL1' in PAR.NORMALIZE:
# normalize event by L1 norm of all traces
w = 0.
for tr in traces:
w += np.linalg.norm(tr.data, ord=1)
for tr in traces:
tr.data /= w
elif 'NormalizeEventsL2' in PAR.NORMALIZE:
# normalize event by L2 norm of all traces
w = 0.
for tr in traces:
w += np.linalg.norm(tr.data, ord=2)
for tr in traces:
tr.data /= w
if 'NormalizeTracesL1' in PAR.NORMALIZE:
# normalize each trace by its L1 norm
for tr in traces:
w = np.linalg.norm(tr.data, ord=1)
if w > 0:
tr.data /= w
elif 'NormalizeTracesL2' in PAR.NORMALIZE:
# normalize each trace by its L2 norm
for tr in traces:
w = np.linalg.norm(tr.data, ord=2)
if w > 0:
tr.data /= w
return traces
def apply_filter_backwards(self, traces):
for tr in traces:
tr.data = np.flip(tr.data)
traces = self.apply_filter()
for tr in traces:
tr.data = np.flip(tr.data)
return traces
# Additional parameter checking
def check_filter(self):
""" Checks filter settings
"""
assert getset(PAR.FILTER) < set([
'Bandpass',
'Lowpass',
'Highpass'])
if PAR.FILTER == 'Bandpass':
if 'FREQMIN' not in PAR:
raise ParameterError('FREQMIN')
if 'FREQMAX' not in PAR:
raise ParameterError('FREQMAX')
assert 0 < PAR.FREQMIN
assert PAR.FREQMIN < PAR.FREQMAX
assert PAR.FREQMAX < np.inf
elif PAR.FILTER == 'Lowpass':
raise NotImplementedError
if 'FREQ' not in PAR:
raise ParameterError('FREQ')
assert 0 < PAR.FREQ <= np.inf
elif PAR.FILTER == 'Highpass':
raise NotImplementedError
if 'FREQ' not in PAR:
raise ParameterError('FREQ')
assert 0 <= PAR.FREQ < np.inf
def check_mute(self):
""" Checks mute settings
"""
if not PAR.MUTE:
return
assert getset(PAR.MUTE) <= set([
'MuteEarlyArrivals',
'MuteLateArrivals',
'MuteShortOffsets',
'MuteLongOffsets'])
if 'MuteEarlyArrivals' in PAR.MUTE:
assert 'MUTE_EARLY_ARRIVALS_SLOPE' in PAR
assert 'MUTE_EARLY_ARRIVALS_CONST' in PAR
assert PAR.MUTE_EARLY_ARRIVALS_SLOPE >= 0.
if 'MuteLateArrivals' in PAR.MUTE:
assert 'MUTE_LATE_ARRIVALS_SLOPE' in PAR
assert 'MUTE_LATE_ARRIVALS_CONST' in PAR
assert PAR.MUTE_LATE_ARRIVALS_SLOPE >= 0.
if 'MuteShortOffsets' in PAR.MUTE:
assert 'MUTE_SHORT_OFFSETS_DIST' in PAR
assert 0 < PAR.MUTE_SHORT_OFFSETS_DIST
if 'MuteLongOffsets' in PAR.MUTE:
assert 'MUTE_LONG_OFFSETS_DIST' in PAR
assert 0 < PAR.MUTE_LONG_OFFSETS_DIST
if 'MuteShortOffsets' not in PAR.MUTE:
setattr(PAR, 'MUTE_SHORT_OFFSETS_DIST', 0.)
if 'MuteLongOffsets' not in PAR.MUTE:
setattr(PAR, 'MUTE_LONG_OFFSETS_DIST', 0.)
def check_normalize(self):
assert getset(PAR.NORMALIZE) < set([
'NormalizeTracesL1',
'NormalizeTracesL2',
'NormalizeEventsL1',
'NormalizeEventsL2'])
# Utility functions
def get_time_scheme(self, traces):
""" FIXME: extract time scheme from trace headers rather than
parameters file.
Note from Alexis Bottero : it is actually better like this in
my opinion because this allows for longer traces to be processed.
Indeed, in su format only 2 bytes are dedicated to the number of
samples which is supposed to be stored as an unsigned int. The
maximum NT which can be stored in the header is then 32762 whereas
there is no limit in principle.
"""
nt = PAR.NT
dt = PAR.DT
t0 = 0.
return nt, dt, t0
def get_network_size(self, traces):
nrec = len(traces)
nsrc = 1
return nrec, nsrc
def get_receiver_coords(self, traces):
if PAR.FORMAT in ['SU', 'su']:
rx = []
ry = []
rz = []
for trace in traces:
rx += [trace.stats.su.trace_header.group_coordinate_x]
ry += [trace.stats.su.trace_header.group_coordinate_y]
rz += [0.]
return rx, ry, rz
else:
raise NotImplementedError
def get_source_coords(self, traces):
if PAR.FORMAT in ['SU', 'su']:
sx = []
sy = []
sz = []
for trace in traces:
sx += [trace.stats.su.trace_header.source_coordinate_x]
sy += [trace.stats.su.trace_header.source_coordinate_y]
sz += [0.]
return sx, sy, sz
else:
raise NotImplementedError
|
|
from ..titanic import digital
from ..titanic import gmpmath
from ..titanic.ops import OP
class MPNum(digital.Digital):
# must be implemented in subclasses
@classmethod
def _select_context(cls, *args, ctx=None):
raise ValueError('virtual method: unimplemented')
@classmethod
def _round_to_context(cls, unrounded, ctx=None, strict=False):
raise ValueError('virtual method: unimplemented')
# most operations
def add(self, other, ctx=None):
ctx = self._select_context(self, other, ctx=ctx)
result = gmpmath.compute(OP.add, self, other, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def sub(self, other, ctx=None):
ctx = self._select_context(self, other, ctx=ctx)
result = gmpmath.compute(OP.sub, self, other, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def mul(self, other, ctx=None):
ctx = self._select_context(self, other, ctx=ctx)
result = gmpmath.compute(OP.mul, self, other, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def div(self, other, ctx=None):
ctx = self._select_context(self, other, ctx=ctx)
result = gmpmath.compute(OP.div, self, other, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def sqrt(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.sqrt, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def fma(self, other1, other2, ctx=None):
ctx = self._select_context(self, other1, other2, ctx=ctx)
result = gmpmath.compute(OP.fma, self, other1, other2, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def neg(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.neg, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def copysign(self, other, ctx=None):
ctx = self._select_context(self, other, ctx=ctx)
result = gmpmath.compute(OP.copysign, self, other, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def fabs(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.fabs, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def fdim(self, other, ctx=None):
# emulated
ctx = self._select_context(self, other, ctx=ctx)
result = gmpmath.compute(OP.sub, self, other, prec=ctx.p)
zero = digital.Digital(negative=False, c=0, exp=0)
if result < zero:
return type(self)(negative=False, c=0, exp=0, inexact=False, rc=0)
else:
# never return negative zero
rounded = self._round_to_context(result, ctx=ctx, strict=True)
return type(self)(rounded, negative=False)
def fmax(self, other, ctx=None):
# emulated
ctx = self._select_context(self, other, ctx=ctx)
if self.isnan:
return self._round_to_context(other, ctx=ctx, strict=False)
elif other.isnan:
return self._round_to_context(self, ctx=ctx, strict=False)
else:
return self._round_to_context(max(self, other), ctx=ctx, strict=False)
def fmin(self, other, ctx=None):
# emulated
ctx = self._select_context(self, other, ctx=ctx)
if self.isnan:
return self._round_to_context(other, ctx=ctx, strict=False)
elif other.isnan:
return self._round_to_context(self, ctx=ctx, strict=False)
else:
return self._round_to_context(min(self, other), ctx=ctx, strict=False)
def fmod(self, other, ctx=None):
ctx = self._select_context(self, other, ctx=ctx)
result = gmpmath.compute(OP.fmod, self, other, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def remainder(self, other, ctx=None):
ctx = self._select_context(self, other, ctx=ctx)
result = gmpmath.compute(OP.remainder, self, other, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def ceil(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.ceil, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def floor(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.floor, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def nearbyint(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.nearbyint, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def round(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.round, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def trunc(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.trunc, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def acos(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.acos, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def acosh(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.acosh, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def asin(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.asin, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def asinh(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.asinh, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def atan(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.atan, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def atan2(self, other, ctx=None):
ctx = self._select_context(self, other, ctx=ctx)
result = gmpmath.compute(OP.atan2, self, other, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def atanh(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.atanh, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def cos(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.cos, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def cosh(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.cosh, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def sin(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.sin, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def sinh(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.sinh, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def tan(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.tan, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def tanh(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.tanh, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def exp_(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.exp, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def exp2(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.exp2, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def expm1(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.expm1, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def log(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.log, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def log10(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.log10, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def log1p(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.log1p, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def log2(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.log2, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def cbrt(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.cbrt, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def hypot(self, other, ctx=None):
ctx = self._select_context(self, other, ctx=ctx)
result = gmpmath.compute(OP.hypot, self, other, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def pow(self, other, ctx=None):
ctx = self._select_context(self, other, ctx=ctx)
if other.is_zero():
# avoid possibly passing nan to gmpmath.compute
return type(self)(negative=False, c=1, exp=0, inexact=False, rc=0)
result = gmpmath.compute(OP.pow, self, other, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def erf(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.erf, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def erfc(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.erfc, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def lgamma(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.lgamma, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def tgamma(self, ctx=None):
ctx = self._select_context(self, ctx=ctx)
result = gmpmath.compute(OP.tgamma, self, prec=ctx.p)
return self._round_to_context(result, ctx=ctx, strict=True)
def isfinite(self):
return not (self.isinf or self.isnan)
# isinf and isnan are properties
# isnormal is implementation specific - override if necessary
def isnormal(self):
return not (
self.is_zero()
or self.isinf
or self.isnan
)
def signbit(self):
return self.negative
|
|
import logging
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.utils.encoding import smart_unicode
from restlib2.http import codes
from restlib2.params import StrParam, IntParam, BoolParam
from modeltree.tree import MODELTREE_DEFAULT_ALIAS, trees
from avocado.events import usage
from avocado.query import pipeline
from .base import FieldBase, is_field_orphaned
from ..pagination import PaginatorResource, PaginatorParametizer
from ...links import patch_response, reverse_tmpl
log = logging.getLogger(__name__)
class FieldValuesParametizer(PaginatorParametizer):
aware = BoolParam(False)
limit = IntParam(10)
tree = StrParam(MODELTREE_DEFAULT_ALIAS, choices=trees)
processor = StrParam('default', choices=pipeline.query_processors)
query = StrParam()
random = IntParam()
class FieldValues(FieldBase, PaginatorResource):
"""Field Values Resource
This resource can be overriden for any field to use a more
performant search implementation.
"""
parametizer = FieldValuesParametizer
def get_base_values(self, request, instance, params):
"Returns the base queryset for this field."
# The `aware` flag toggles the behavior of the distribution by making
# relative to the applied context or none
if params['aware']:
context = self.get_context(request)
else:
context = self.get_context(request, attrs={})
return context.apply(queryset=instance.model.objects.all())
def get_all_values(self, request, instance, queryset):
"Returns all distinct values for this field."
results = []
for value, label in instance.choices(queryset=queryset):
results.append({
'label': label,
'value': value,
})
return results
def get_search_values(self, request, instance, query, queryset):
"""
Performs a search on the underlying data for a field.
This method can be overridden to use an alternate search
implementation.
"""
results = []
value_labels = instance.value_labels(queryset=queryset)
for value in instance.search(query, queryset=queryset):
results.append({
'label': value_labels.get(value, smart_unicode(value)),
'value': value,
})
return results
def get_random_values(self, request, instance, random, queryset):
"""
Returns a random set of value/label pairs.
This is useful for pre-populating documents or form fields with
example data.
"""
values = instance.random(random, queryset=queryset)
results = []
for value in values:
results.append({
'label': instance.get_label(value, queryset=queryset),
'value': value,
})
return results
def get_link_templates(self, request):
uri = request.build_absolute_uri
return {
'parent': reverse_tmpl(
uri, 'serrano:field', {'pk': (int, 'parent_id')})
}
def get(self, request, pk):
instance = self.get_object(request, pk=pk)
if is_field_orphaned(instance):
data = {
'message': 'Orphaned fields do not support values calls.'
}
return self.render(
request, data, status=codes.unprocessable_entity)
params = self.get_params(request)
if params['aware']:
context = self.get_context(request)
else:
context = None
QueryProcessor = pipeline.query_processors[params['processor']]
processor = QueryProcessor(tree=instance.model, context=context)
queryset = processor.get_queryset(request=request)
if params['random']:
# In the case that the queryset contains a population smaller than
# the number of random items being requested, a ValueError will be
# triggered. Instead of passing the error on to the client, we
# simply return all the possible values.
try:
return self.get_random_values(
request, instance, params['random'], queryset)
except ValueError:
return instance.values(queryset=queryset)
page = params['page']
limit = params['limit']
# If a query term is supplied, perform the icontains search.
if params['query']:
usage.log('items', instance=instance, request=request, data={
'query': params['query'],
})
values = self.get_search_values(
request, instance, params['query'], queryset)
else:
values = self.get_all_values(request, instance, queryset)
# No page specified, return everything.
if page is None:
return values
paginator = self.get_paginator(values, limit=limit)
page = paginator.page(page)
# Get paginator-based response.
data = self.get_page_response(request, paginator, page)
data.update({
'items': page.object_list,
})
# Add links.
path = reverse('serrano:field-values', kwargs={'pk': pk})
links = self.get_page_links(request, path, page, extra=params)
templates = self.get_link_templates(request)
response = self.render(request, content=data)
return patch_response(request, response, links, templates)
def post(self, request, pk):
instance = self.get_object(request, pk=pk)
params = self.get_params(request)
if not request.data:
data = {
'message': 'Error parsing data',
}
return self.render(request, data,
status=codes.unprocessable_entity)
if isinstance(request.data, dict):
array = [request.data]
else:
array = request.data
values = []
labels = []
array_map = {}
# Separate out the values and labels for the lookup. Track indexes
# maintain order of array
for i, datum in enumerate(array):
# Value takes precedence over label if supplied.
if 'value' in datum:
array_map[i] = 'value'
values.append(datum['value'])
elif 'label' in datum:
array_map[i] = 'label'
labels.append(datum['label'])
else:
data = {
'message': 'Error parsing value or label'
}
return self.render(request, data,
status=codes.unprocessable_entity)
value_field_name = instance.field_name
label_field_name = instance.label_field.name
# Note, this return a context-aware or naive queryset depending
# on params. Get the value and label fields so they can be filled
# in below.
queryset = self.get_base_values(request, instance, params)\
.values_list(value_field_name, label_field_name)
lookup = Q()
# Validate based on the label.
if labels:
lookup |= Q(**{'{0}__in'.format(label_field_name): labels})
if values:
lookup |= Q(**{'{0}__in'.format(value_field_name): values})
results = queryset.filter(lookup)
value_labels = dict(results)
label_values = dict([(v, k) for k, v in value_labels.items()])
for i, datum in enumerate(array):
if array_map[i] == 'label':
valid = datum['label'] in label_values
if valid:
value = label_values[datum['label']]
else:
value = datum['label']
datum['valid'] = valid
datum['value'] = value
else:
valid = datum['value'] in value_labels
if valid:
label = value_labels[datum['value']]
else:
label = smart_unicode(datum['value'])
datum['valid'] = valid
datum['label'] = label
usage.log('validate', instance=instance, request=request, data={
'count': len(array),
})
# Return the augmented data.
return request.data
|
|
#! /usr/env/python
"""
This module attempts to "component-ify" GT's Fastscape stream power erosion.
Created DEJH, March 2014.
"""
from __future__ import print_function
import numpy
import warnings
from landlab import ModelParameterDictionary, Component
from landlab.core.model_parameter_dictionary import MissingKeyError, \
ParameterValueError
from landlab.utils.decorators import use_file_name_or_kwds
from landlab.field.scalar_data_fields import FieldError
from scipy.optimize import newton, fsolve
UNDEFINED_INDEX = -1
class FastscapeEroder(Component):
'''
This class uses the Braun-Willett Fastscape approach to calculate the
amount of erosion at each node in a grid, following a stream power
framework. This should allow it to be stable against larger timesteps
than an explicit stream power scheme.
Stream power erosion is implemented as::
E = K * (rainfall_intensity*A)**m * S**n - threshold_sp,
if K * A**m * S**n > threshold_sp, and::
E = 0,
if K * A**m * S**n <= threshold_sp.
This module assumes you have already run
:func:`landlab.components.flow_routing.route_flow_dn.FlowRouter.route_flow`
in the same timestep. It looks for 'flow__upstream_node_order',
'flow__link_to_receiver_node', 'drainage_area', 'flow__receiver_node', and
'topographic__elevation' at the nodes in the grid. 'drainage_area' should
be in area upstream, not volume (i.e., set runoff_rate=1.0 when calling
FlowRouter.route_flow).
The primary method of this class is :func:`run_one_step`.
Construction::
FastscapeEroder(grid, K_sp=None, m_sp=0.5, n_sp=1., threshold_sp=0.,
rainfall_intensity=1.)
Parameters
----------
grid : ModelGrid
A grid.
K_sp : float, array, or field name
K in the stream power equation (units vary with other parameters).
m_sp : float, optional
m in the stream power equation (power on drainage area).
n_sp : float, optional, ~ 0.5<n_sp<4.
n in the stream power equation (power on slope).
Performance will be VERY degraded if n < 1.
threshold_sp : float, array, or field name
The threshold stream power.
rainfall_intensity : float; optional
Modifying factor on drainage area to convert it to a true water
volume flux in (m/time). i.e., E = K * (r_i*A)**m * S**n. For a time
varying rainfall intensity, pass rainfall_intensity_if_used to
`run_one_step`. For a spatially variable rainfall, use the
StreamPowerEroder component.
Examples
--------
>>> import numpy as np
>>> from landlab import RasterModelGrid
>>> from landlab import CLOSED_BOUNDARY, FIXED_VALUE_BOUNDARY
>>> from landlab.components import FlowRouter
>>> mg = RasterModelGrid((5, 5), 10.)
>>> z = np.array([7., 7., 7., 7., 7.,
... 7., 5., 3.2, 6., 7.,
... 7., 2., 3., 5., 7.,
... 7., 1., 1.9, 4., 7.,
... 7., 0., 7., 7., 7.])
>>> z = mg.add_field('node', 'topographic__elevation', z)
>>> fr = FlowRouter(mg)
>>> sp = FastscapeEroder(mg, K_sp=1.)
>>> fr.run_one_step()
>>> sp.run_one_step(dt=1.)
>>> z # doctest: +NORMALIZE_WHITESPACE
array([ 7. , 7. , 7. , 7. , 7. ,
7. , 2.92996598, 2.02996598, 4.01498299, 7. ,
7. , 0.85993197, 1.87743897, 3.28268321, 7. ,
7. , 0.28989795, 0.85403051, 2.42701526, 7. ,
7. , 0. , 7. , 7. , 7. ])
>>> mg2 = RasterModelGrid((3, 7), 1.)
>>> z = np.array(mg2.node_x**2.)
>>> z = mg2.add_field('node', 'topographic__elevation', z)
>>> mg2.status_at_node[mg2.nodes_at_left_edge] = FIXED_VALUE_BOUNDARY
>>> mg2.status_at_node[mg2.nodes_at_top_edge] = CLOSED_BOUNDARY
>>> mg2.status_at_node[mg2.nodes_at_bottom_edge] = CLOSED_BOUNDARY
>>> mg2.status_at_node[mg2.nodes_at_right_edge] = CLOSED_BOUNDARY
>>> fr2 = FlowRouter(mg2)
>>> sp2 = FastscapeEroder(mg2, K_sp=0.1, m_sp=0., n_sp=2.,
... threshold_sp=2.)
>>> fr2.run_one_step()
>>> sp2.run_one_step(dt=10.)
>>> z.reshape((3, 7))[1, :] # doctest: +NORMALIZE_WHITESPACE
array([ 0. , 1. , 4. , 8.52493781,
13.29039716, 18.44367965, 36. ])
>>> mg3 = RasterModelGrid((3, 7), 1.)
>>> z = np.array(mg3.node_x**2.)
>>> z = mg3.add_field('node', 'topographic__elevation', z)
>>> mg3.status_at_node[mg3.nodes_at_left_edge] = FIXED_VALUE_BOUNDARY
>>> mg3.status_at_node[mg3.nodes_at_top_edge] = CLOSED_BOUNDARY
>>> mg3.status_at_node[mg3.nodes_at_bottom_edge] = CLOSED_BOUNDARY
>>> mg3.status_at_node[mg3.nodes_at_right_edge] = CLOSED_BOUNDARY
>>> fr3 = FlowRouter(mg3)
>>> K_field = mg3.ones('node') # K can be a field
>>> sp3 = FastscapeEroder(mg3, K_sp=K_field, m_sp=1., n_sp=0.6,
... threshold_sp=mg3.node_x,
... rainfall_intensity=2.)
>>> fr3.run_one_step()
>>> sp3.run_one_step(1.)
>>> z.reshape((3, 7))[1, :] # doctest: +NORMALIZE_WHITESPACE
array([ 0. , 0.0647484 , 0.58634455, 2.67253503,
8.49212152, 20.92606987, 36. ])
>>> previous_z = z.copy()
>>> sp3.run_one_step(1., rainfall_intensity_if_used=0.)
>>> np.allclose(z, previous_z)
True
'''
_name = 'FastscapeEroder'
_input_var_names = (
'topographic__elevation',
'drainage_area',
'flow__link_to_receiver_node',
'flow__upstream_node_order',
'flow__receiver_node',
)
_output_var_names = (
'topographic__elevation',
)
_var_units = {
'topographic__elevation': 'm',
'drainage_area': 'm**2',
'flow__link_to_receiver_node': '-',
'flow__upstream_node_order': '-',
'flow__receiver_node': '-',
}
_var_mapping = {
'topographic__elevation': 'node',
'drainage_area': 'node',
'flow__link_to_receiver_node': 'node',
'flow__upstream_node_order': 'node',
'flow__receiver_node': 'node',
}
_var_doc = {
'topographic__elevation': 'Land surface topographic elevation',
'drainage_area':
"Upstream accumulated surface area contributing to the node's "
"discharge",
'flow__link_to_receiver_node':
'ID of link downstream of each node, which carries the discharge',
'flow__upstream_node_order':
'Node array containing downstream-to-upstream ordered list of '
'node IDs',
'flow__receiver_node':
'Node array of receivers (node that receives flow from current '
'node)',
}
@use_file_name_or_kwds
def __init__(self, grid, K_sp=None, m_sp=0.5, n_sp=1., threshold_sp=0.,
rainfall_intensity=1., **kwds):
"""
Initialize the Fastscape stream power component. Note: a timestep,
dt, can no longer be supplied to this component through the input file.
It must instead be passed directly to the run method.
Parameters
----------
grid : ModelGrid
A grid.
K_sp : float, array, or field name
K in the stream power equation (units vary with other parameters).
m_sp : float, optional
m in the stream power equation (power on drainage area).
n_sp : float, optional
n in the stream power equation (power on slope).
rainfall intensity : float, array, or field name; optional
Modifying factor on drainage area to convert it to a true water
volume flux in (m/time). i.e., E = K * (r_i*A)**m * S**n
"""
self._grid = grid
self.K = K_sp # overwritten below in special cases
self.m = float(m_sp)
self.n = float(n_sp)
if type(threshold_sp) in (float, int):
self.thresholds = float(threshold_sp)
else:
if type(threshold_sp) is str:
self.thresholds = self.grid.at_node[threshold_sp]
else:
self.thresholds = threshold_sp
assert self.thresholds.size == self.grid.number_of_nodes
# make storage variables
self.A_to_the_m = grid.zeros(at='node')
self.alpha = grid.empty(at='node')
self.alpha_by_flow_link_lengthtothenless1 = numpy.empty_like(
self.alpha)
try:
self.grid._diagonal_links_at_node # calc number of diagonal links
except AttributeError:
pass # was not a raster
if self.K is None:
raise ValueError('K_sp must be set as a float, node array, or ' +
'field name. It was None.')
# now handle the inputs that could be float, array or field name:
# some support here for old-style inputs
if type(K_sp) is str:
if K_sp == 'array':
self.K = None
else:
self.K = self._grid.at_node[K_sp]
elif type(K_sp) in (float, int): # a float
self.K = float(K_sp)
elif len(K_sp) == self.grid.number_of_nodes:
self.K = numpy.array(K_sp)
else:
raise TypeError('Supplied type of K_sp ' +
'was not recognised, or array was ' +
'not nnodes long!')
if type(rainfall_intensity) is str:
raise ValueError('This component can no longer handle ' +
'spatially variable rainfall. Use ' +
'StreamPowerEroder.')
if rainfall_intensity == 'array':
self._r_i = None
else:
self._r_i = self._grid.at_node[rainfall_intensity]
elif type(rainfall_intensity) in (float, int): # a float
self._r_i = float(rainfall_intensity)
elif len(rainfall_intensity) == self.grid.number_of_nodes:
raise ValueError('This component can no longer handle ' +
'spatially variable rainfall. Use ' +
'StreamPowerEroder.')
self._r_i = numpy.array(rainfall_intensity)
else:
raise TypeError('Supplied type of rainfall_' +
'intensity was not recognised!')
# We now forbid changing of the field name
if 'value_field' in kwds.keys():
raise ValueError('This component can no longer support variable' +
'field names. Use "topographic__elevation".')
def erode(self, grid_in, dt=None, K_if_used=None, flooded_nodes=None,
rainfall_intensity_if_used=None):
"""
This method implements the stream power erosion, following the Braun-
Willett (2013) implicit Fastscape algorithm. This should allow it to
be stable against larger timesteps than an explicit stream power
scheme.
This driving method for this component is now superceded by the new,
standardized wrapper :func:`run_one_step`, but is retained for
back compatibility.
Set 'K_if_used' as a field name or nnodes-long array if you set K_sp as
'array' during initialization.
It returns the grid, in which it will have modified the value of
*value_field*, as specified in component initialization.
Parameters
----------
grid_in : a grid
This is a dummy argument maintained for component back-
compatibility. It is superceded by the copy of the grid passed
during initialization.
dt : float
Time-step size. If you are calling the deprecated function
:func:`gear_timestep`, that method will supercede any value
supplied here.
K_if_used : array (optional)
Set this to an array if you set K_sp to 'array' in your input file.
flooded_nodes : ndarray of int (optional)
IDs of nodes that are flooded and should have no erosion. If not
provided but flow has still been routed across depressions, erosion
may still occur beneath the apparent water level (though will
always still be positive).
rainfall_intensity_if_used : float or None (optional)
Supply to drive this component with a time-varying spatially
constant rainfall.
Returns
-------
grid
A reference to the grid.
"""
self.alpha = numpy.zeros(self._grid.number_of_nodes)
self.alpha_by_flow_link_lengthtothenless1 = numpy.zeros(self._grid.number_of_nodes)
upstream_order_IDs = self._grid['node']['flow__upstream_node_order']
z = self._grid['node']['topographic__elevation']
defined_flow_receivers = numpy.not_equal(self._grid['node'][
'flow__link_to_receiver_node'], UNDEFINED_INDEX)
flow_link_lengths = self._grid._length_of_link_with_diagonals[
self._grid['node']['flow__link_to_receiver_node'][
defined_flow_receivers]]
# make arrays from input the right size
if type(self.K) is numpy.ndarray:
K_here = self.K[defined_flow_receivers]
else:
K_here = self.K
if rainfall_intensity_if_used is not None:
assert type(rainfall_intensity_if_used) in (float, int)
r_i_here = float(rainfall_intensity_if_used)
else:
r_i_here = self._r_i
if dt is None:
dt = self.dt
assert dt is not None, ('Fastscape component could not find a dt to ' +
'use. Pass dt to the run_one_step() method.')
if self.K is None: # "old style" setting of array
assert K_if_used is not None
self.K = K_if_used
numpy.power(self._grid['node']['drainage_area'], self.m,
out=self.A_to_the_m)
self.alpha[defined_flow_receivers] = r_i_here**self.m * K_here * dt * \
self.A_to_the_m[defined_flow_receivers] / flow_link_lengths
flow_receivers = self._grid['node']['flow__receiver_node']
n_nodes = upstream_order_IDs.size
alpha = self.alpha
# Handle flooded nodes, if any (no erosion there)
if flooded_nodes is not None:
alpha[flooded_nodes] = 0.
else:
reversed_flow = z < z[flow_receivers]
# this check necessary if flow has been routed across depressions
alpha[reversed_flow] = 0.
self.alpha_by_flow_link_lengthtothenless1[
defined_flow_receivers] = (alpha[defined_flow_receivers] /
flow_link_lengths**(self.n - 1.))
alpha_divided = self.alpha_by_flow_link_lengthtothenless1
n = float(self.n)
threshdt = self.thresholds * dt
if type(self.thresholds) is float:
from .cfuncs import erode_with_link_alpha_fixthresh
erode_with_link_alpha_fixthresh(upstream_order_IDs, flow_receivers,
threshdt, alpha_divided, n, z)
else:
from .cfuncs import erode_with_link_alpha_varthresh
erode_with_link_alpha_varthresh(upstream_order_IDs, flow_receivers,
threshdt, alpha_divided, n, z)
# # This replicates the cython for testing:
# for i in range(upstream_order_IDs.size):
# src_id = upstream_order_IDs[i]
# dst_id = flow_receivers[src_id]
# thresh = threshdt[i]
# if src_id != dst_id:
# next_z = z[src_id]
# prev_z = 0.
# while True:
# #for j in range(2):
# z_diff = next_z - z[dst_id]
# f = alpha_divided[src_id] * pow(z_diff, n - 1.)
# # if z_diff -> 0, pow -> nan (in reality, inf)
# # print (f, prev_z, next_z, z_diff, z[dst_id])
# next_z = next_z - ((next_z - z[src_id] + (
# f*z_diff - thresh).clip(0.)) / (1. + n * f))
# if next_z < z[dst_id]:
# next_z = z[dst_id] + 1.e-15
# # ^maintain connectivity
# if next_z != 0.:
# if (numpy.fabs((next_z - prev_z)/next_z) <
# 1.48e-08) or (n == 1.):
# break
# else:
# break
# prev_z = next_z
# if next_z < z[src_id]:
# z[src_id] = next_z
return self._grid
def run_one_step(self, dt, flooded_nodes=None,
rainfall_intensity_if_used=None, **kwds):
"""
This method implements the stream power erosion across one time
interval, dt, following the Braun-Willett (2013) implicit Fastscape
algorithm.
This follows Landlab standardized component design, and supercedes the
old driving method :func:`erode`.
Parameters
----------
dt : float
Time-step size
flooded_nodes : ndarray of int (optional)
IDs of nodes that are flooded and should have no erosion. If not
provided but flow has still been routed across depressions, erosion
may still occur beneath the apparent water level (though will
always still be positive).
rainfall_intensity_if_used : float or None (optional)
Supply to drive this component with a time-varying spatially
constant rainfall.
"""
self.erode(grid_in=self._grid, dt=dt, flooded_nodes=flooded_nodes,
rainfall_intensity_if_used=rainfall_intensity_if_used)
|
|
import logging
import os
import shutil
import tempfile
from crawler_exceptions import CrawlError, CrawlUnsupportedPackageManager
from utils import osinfo
from utils.features import PackageFeature
from utils.misc import subprocess_run
logger = logging.getLogger('crawlutils')
def get_dpkg_packages(
root_dir='/',
dbpath='var/lib/dpkg',
installed_since=0):
if os.path.isabs(dbpath):
logger.warning(
'dbpath: ' +
dbpath +
' is defined absolute. Ignoring prefix: ' +
root_dir +
'.')
# Update for a different route.
dbpath = os.path.join(root_dir, dbpath)
output = subprocess_run(['dpkg-query', '-W',
'--admindir={0}'.format(dbpath),
'-f=${Package}|${Version}'
'|${Architecture}|${Installed-Size}\n'],
shell=False)
dpkglist = output.strip('\n')
if dpkglist:
for dpkginfo in dpkglist.split('\n'):
(name, version, architecture, size) = dpkginfo.split(r'|')
# dpkg does not provide any installtime field
# feature_key = '{0}/{1}'.format(name, version) -->
# changed to below per Suriya's request
feature_key = '{0}'.format(name, version)
yield (feature_key, PackageFeature(None, name,
size, version,
architecture))
def get_rpm_packages(
root_dir='/',
dbpath='var/lib/rpm',
installed_since=0,
reload_needed=False):
if os.path.isabs(dbpath):
logger.warning(
'dbpath: ' +
dbpath +
' is defined absolute. Ignoring prefix: ' +
root_dir +
'.')
# update for a different route
dbpath = os.path.join(root_dir, dbpath)
try:
if reload_needed:
reloaded_db_dir = tempfile.mkdtemp()
_rpm_reload_db(root_dir, dbpath, reloaded_db_dir)
dbpath = reloaded_db_dir
output = subprocess_run(['rpm',
'--dbpath',
dbpath,
'-qa',
'--queryformat',
'%{installtime}|%{name}|%{version}'
'-%{release}|%{arch}|%{size}\n'],
shell=False,
ignore_failure=True)
# We ignore failures because sometimes rpm returns rc=1 but still
# outputs all the data.
rpmlist = output.strip('\n')
finally:
if reload_needed:
logger.debug('Deleting directory: %s' % (reloaded_db_dir))
shutil.rmtree(reloaded_db_dir)
if rpmlist:
for rpminfo in rpmlist.split('\n'):
(installtime, name, version, architecture, size) = \
rpminfo.split(r'|')
"""
if int(installtime) <= installed_since: --> this
barfs for sth like: 1376416422. Consider try: xxx
except ValueError: pass
"""
if installtime <= installed_since:
continue
"""
feature_key = '{0}/{1}'.format(name, version) -->
changed to below per Suriya's request
"""
feature_key = '{0}'.format(name, version)
yield (feature_key,
PackageFeature(installtime,
name, size, version, architecture))
def _rpm_reload_db(
root_dir='/',
dbpath='var/lib/rpm',
reloaded_db_dir='/tmp/'):
"""
Dumps and reloads the rpm database.
Returns the path to the new rpm database, or raises RuntimeError if the
dump and load commands failed.
"""
try:
dump_dir = tempfile.mkdtemp()
subprocess_run(['/usr/bin/db_dump',
os.path.join(dbpath, 'Packages'),
'-f',
os.path.join(dump_dir, 'Packages')],
shell=False)
subprocess_run(['/usr/bin/db_load',
'-f',
os.path.join(dump_dir, 'Packages'),
os.path.join(reloaded_db_dir, 'Packages')],
shell=False)
finally:
logger.debug('Deleting directory: %s' % (dump_dir))
shutil.rmtree(dump_dir)
return reloaded_db_dir
# from UK crawler codebase
def apk_parser(filename):
try:
db_contents = open(filename).read()
packages = db_contents.split('\n\n')
logger.debug('Found {} APK packages'.format(len(packages)))
for package in packages:
if package:
attributes = package.split('\n')
name = ""
version = ""
architecture = ""
size = ""
for attribute in attributes:
if (attribute.startswith('P:')):
name = attribute[2:]
elif (attribute.startswith('V:')):
version = attribute[2:]
elif (attribute.startswith('A:')):
architecture = attribute[2:]
elif (attribute.startswith('S:')):
size = attribute[2:]
yield (name, PackageFeature(None, name,
size, version,
architecture))
except IOError as e:
logger.error('Failed to read APK database to obtain packages. '
'Check if %s is present. [Exception: %s: %s]'
' ' % (filename, type(e).__name__, e.strerror))
raise
def get_apk_packages(
root_dir='/',
dbpath='lib/apk/db'):
if os.path.isabs(dbpath):
logger.warning(
'dbpath: ' +
dbpath +
' is defined absolute. Ignoring prefix: ' +
root_dir +
'.')
# Update for a different route.
dbpath = os.path.join(root_dir, dbpath)
for feature_key, package_feature in apk_parser(
os.path.join(dbpath, 'installed')):
yield (feature_key, package_feature)
def crawl_packages(
dbpath=None,
root_dir='/',
installed_since=0,
reload_needed=True):
# package attributes: ["installed", "name", "size", "version"]
logger.debug('Crawling Packages')
try:
pkg_manager = _get_package_manager(root_dir)
if pkg_manager == 'dpkg':
dbpath = dbpath or 'var/lib/dpkg'
for (key, feature) in get_dpkg_packages(
root_dir, dbpath, installed_since):
yield (key, feature, 'package')
elif pkg_manager == 'rpm':
dbpath = dbpath or 'var/lib/rpm'
for (key, feature) in get_rpm_packages(
root_dir, dbpath, installed_since, reload_needed):
yield (key, feature, 'package')
elif pkg_manager == 'apk':
dbpath = dbpath or 'lib/apk/db'
for (key, feature) in get_apk_packages(
root_dir, dbpath):
yield (key, feature, 'package')
else:
logger.warning('Unsupported package manager for Linux distro')
except Exception as e:
logger.error('Error crawling packages',
exc_info=True)
raise CrawlError(e)
def _get_package_manager(root_dir):
result = osinfo.get_osinfo(mount_point=root_dir)
if result:
os_distro = result['os']
else:
raise CrawlUnsupportedPackageManager()
pkg_manager = None
if os_distro in ['ubuntu', 'debian']:
pkg_manager = 'dpkg'
elif os_distro in ['redhat', 'red hat', 'rhel', 'fedora', 'centos']:
pkg_manager = 'rpm'
elif os_distro in ['alpine']:
pkg_manager = 'apk'
elif os.path.exists(os.path.join(root_dir, 'var/lib/dpkg')):
pkg_manager = 'dpkg'
elif os.path.exists(os.path.join(root_dir, 'var/lib/rpm')):
pkg_manager = 'rpm'
return pkg_manager
|
|
import math
import os
import threading
from collections import defaultdict
from typing import Dict
import copy
from twisted.internet.address import IPv4Address
import bptc
from bptc.data.consensus import divide_rounds, decide_fame, find_order
from bptc.data.event import Event, Parents
from bptc.data.member import Member
from bptc.utils.toposort import toposort
from bptc.data.transaction import MoneyTransaction, TransactionStatus, PublishNameTransaction
class Hashgraph:
"""
The Hashgraph - storing the events of all nodes
"""
def __init__(self, me, debug_mode=False):
self.lock = threading.RLock()
# Member: A reference to the current user. For convenience (e.g. signing)
self.me = me
self.debug_mode = debug_mode
# {member-id => Member}: All members we know
if me is not None:
self.known_members = {me.id: me}
# {event-hash => event}: Dictionary mapping hashes to events
self.lookup_table = {}
# {event-hash}: Events for which the final order has not yet been determined
self.unordered_events = set()
# [event-hash]: Final order of events
self.ordered_events = []
self.next_ordered_event_idx_to_process = 0
self.idx = {}
# {round-num}: rounds where fame is fully decided
self.rounds_with_decided_fame = set()
# {round-num => {member-pk => event-hash}}:
self.witnesses = defaultdict(dict)
# {event-hash => set(event-hash)}: Cache for event's self-children (used for fast fork check)
self.self_children_cache = defaultdict(set)
# set(member-id): A set of member who forked. Members who forked have no visible events.
self.fork_blacklist = set()
@property
def total_stake(self) -> int:
"""
:return: The total stake in the hashgraph
"""
return sum([member.stake for _, member in self.known_members.items()])
@property
def supermajority_stake(self) -> int:
"""
:return: The stake needed for a supermajority (2/3 of total)
"""
return int(math.floor(2 * self.total_stake / 3))
def get_unknown_events_of(self, member: Member) -> Dict[str, Event]:
"""
Returns the presumably unknown events of a given member, in the same format as lookup_table
:param member: The member for which to return unknown events
:return: Dictionary mapping hashes to events
"""
result = dict(self.lookup_table)
head = member.head
if head is None:
return result
to_visit = {head}
visited = set()
while len(to_visit) > 0:
event_id = to_visit.pop()
if event_id not in visited:
event = result[event_id]
del result[event_id]
if event.parents.self_parent is not None:
to_visit.add(event.parents.self_parent)
if event.parents.other_parent is not None:
to_visit.add(event.parents.other_parent)
visited.add(event_id)
return result
def add_own_event(self, event: Event, calculate_consensus: bool = False):
"""
Adds an own event to the hashgraph
:param event: The event to be added
:param calculate_consensus: Whether the consensus should be calculated immediately
:return: None
"""
# Sign event body
event.sign(self.me.signing_key)
# Add event
self.add_event(event)
# Only do consensus if this is the first event
if calculate_consensus:
divide_rounds(self, [event])
decide_fame(self)
find_order(self)
self.process_ordered_events()
def add_event(self, event: Event):
# Set the event's correct height
if event.parents.self_parent:
event.height = self.lookup_table[event.parents.self_parent].height + 1
# Add event to graph
self.lookup_table[event.id] = event
# Update caches
self.unordered_events.add(event.id)
if self.known_members[event.verify_key].head is None or \
event.height > self.lookup_table[self.known_members[event.verify_key].head].height:
self.known_members[event.verify_key].head = event.id
if event.parents.self_parent is not None:
self.self_children_cache[event.parents.self_parent].add(event.id)
if len(self.self_children_cache[event.parents.self_parent]) > 1:
# We just added a fork
bptc.logger.warn("A fork was created! Blacklisting member and clearing visibility caches.")
# Blacklist the member who forked
self.fork_blacklist.add(event.verify_key)
# Visibility for events could have changed - throw away the caches
for e in self.lookup_table.values():
e.can_see_cache.clear()
def process_events(self, from_member: Member, events: Dict[str, Event]) -> None:
"""
Processes a list of events
:param from_member: The member from whom the events were received
:param events: The events to be processed
:return: None
"""
events = copy.deepcopy(events)
bptc.logger.debug("Processing {} events from {}...".format(len(events), from_member.verify_key[:6]))
# Only deal with valid events
events = filter_valid_events(events)
events_toposorted = toposort(events)
# Learn about other members
self.learn_members_from_events(events)
# Add all new events in topological order and check parent pointer
new_events = {}
for event in events_toposorted:
if event.id not in self.lookup_table:
if event.parents.self_parent is not None and event.parents.self_parent not in self.lookup_table:
bptc.logger.error('Self parent {} of {} not known. Ignore all data.'.
format(event.parents.self_parent[:6], event.id[:6]))
return
if event.parents.other_parent is not None and event.parents.other_parent not in self.lookup_table:
bptc.logger.error('Other parent {} of {} not known. Ignore all data'.
format(event.parents.other_parent[:6], event.id[:6]))
return
new_events[event.id] = event
self.add_event(event)
# Create a new event for the gossip
event = Event(self.me.verify_key, None, Parents(self.me.head, from_member.head))
self.add_own_event(event)
new_events[event.id] = event
# Figure out fame, order, etc.
divide_rounds(self, toposort(new_events))
decide_fame(self)
find_order(self)
self.process_ordered_events()
# Debug mode writes the DB to a file every 100 events.
if self.debug_mode:
number_events = (len(self.lookup_table) // 100) * 100
# Don't store when there are not enough events or it would overwrite
# the last temporary db
if number_events > 0 and number_events > self.debug_mode:
bptc.logger.debug('Store intermediate results containing about {} events'.format(number_events))
from bptc.data.db import DB
DB.save(self, temp=True)
self.debug_mode = (len(self.lookup_table) // 100) * 100
def learn_members_from_events(self, events: Dict[str, Event]) -> None:
"""
Goes through a list of events and learns their creators if they are not already known
:param events: The list of events
:return: None
"""
for event in events.values():
if event.verify_key not in self.known_members:
self.known_members[event.verify_key] = Member(event.verify_key, None)
def process_ordered_events(self):
for event_id in self.ordered_events[self.next_ordered_event_idx_to_process:len(self.ordered_events)]:
event = self.lookup_table[event_id]
if event.data is None:
continue
for transaction in event.data:
sender = self.known_members[event.verify_key]
if isinstance(transaction, MoneyTransaction):
receiver = self.known_members[transaction.receiver]
# Check if the sender has the funds
if sender.account_balance < transaction.amount or transaction.amount < 0:
transaction.status = TransactionStatus.DENIED
else:
sender.account_balance -= transaction.amount
receiver.account_balance += transaction.amount
transaction.status = TransactionStatus.CONFIRMED
elif isinstance(transaction, PublishNameTransaction):
sender.name = transaction.name
self.next_ordered_event_idx_to_process = len(self.ordered_events)
def parse_transaction(self, event, transaction, plain=False):
receiver = self.known_members[transaction.receiver].formatted_name if \
transaction.receiver in self.known_members else transaction.receiver
sender = self.known_members[event.verify_key].formatted_name if \
event.verify_key in self.known_members else event.verify_key
status = TransactionStatus.text_for_value(transaction.status)
is_received = transaction.receiver == self.me.to_verifykey_string()
amount = transaction.amount
comment = transaction.comment
time = event.time
rec = {
'receiver': receiver,
'sender': sender,
'amount': amount,
'comment': comment,
'time': time,
'status': status,
'is_received': is_received,
}
format_string = '{} [b]{} BPTC[/b] {} [b]{}[/b] ({}){}'
if plain:
format_string = '{} {} BPTC {} {} ({}){}'
rec['formatted'] = format_string.format(
'Received' if is_received else 'Sent',
amount,
'from' if rec['is_received'] else 'to',
sender if rec['is_received'] else receiver,
status,
'\n"{}"'.format(comment) if comment else '',
).replace('\n', ' - ' if plain else '\n')
return rec
def get_relevant_transactions(self, plain=False, show_all=False):
# Load transactions belonging to this member
transactions = []
events = list(self.lookup_table.values())
for e in events:
for t in e.data or []:
if isinstance(t, MoneyTransaction):
if show_all or self.me.to_verifykey_string() in [e.verify_key, t.receiver]:
transactions.append(self.parse_transaction(e, t, plain))
return sorted(transactions, key=lambda x: x['time'], reverse=True)
def filter_valid_events(events: Dict[str, Event]) -> Dict[str, Event]:
"""
Goes through a dict of events and returns a dict containing only the valid ones
:param events: The dict to be filtered
:return: A dict containing only valid events
"""
result = dict()
for event_id, event in events.items():
if event.has_valid_signature:
result[event_id] = event
else:
bptc.logger.warn("Event had invalid signature: {}".format(event))
return result
def init_hashgraph(app):
"""Loads the hashgraph from file or creates a new one, if the file doesn't exist."""
from bptc.data.db import DB
from bptc.data.network import Network
# Try to load the Hashgraph from the database
hashgraph = DB.load_hashgraph(os.path.join(app.cl_args.output, 'data.db'))
hashgraph.debug_mode = app.cl_args.debug
# Create a new hashgraph if it could not be loaded
if hashgraph is None or hashgraph.me is None:
me = Member.create()
me.address = IPv4Address("TCP", bptc.ip, bptc.port)
hashgraph = Hashgraph(me, app.cl_args.debug)
app.network = Network(hashgraph, create_initial_event=True)
else:
app.network = Network(hashgraph, create_initial_event=False)
|
|
# -*- coding: utf-8 -*-
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from collections import OrderedDict
from distutils import util
import os
import re
from typing import Dict, Optional, Sequence, Tuple, Type, Union
from google.api_core import client_options as client_options_lib # type: ignore
from google.api_core import exceptions as core_exceptions # type: ignore
from google.api_core import gapic_v1 # type: ignore
from google.api_core import retry as retries # type: ignore
from google.auth import credentials as ga_credentials # type: ignore
from google.auth.transport import mtls # type: ignore
from google.auth.transport.grpc import SslCredentials # type: ignore
from google.auth.exceptions import MutualTLSChannelError # type: ignore
from google.oauth2 import service_account # type: ignore
from google.ads.googleads.v8.resources.types import campaign_simulation
from google.ads.googleads.v8.services.types import campaign_simulation_service
from .transports.base import (
CampaignSimulationServiceTransport,
DEFAULT_CLIENT_INFO,
)
from .transports.grpc import CampaignSimulationServiceGrpcTransport
class CampaignSimulationServiceClientMeta(type):
"""Metaclass for the CampaignSimulationService client.
This provides class-level methods for building and retrieving
support objects (e.g. transport) without polluting the client instance
objects.
"""
_transport_registry = (
OrderedDict()
) # type: Dict[str, Type[CampaignSimulationServiceTransport]]
_transport_registry["grpc"] = CampaignSimulationServiceGrpcTransport
def get_transport_class(
cls, label: str = None,
) -> Type[CampaignSimulationServiceTransport]:
"""Return an appropriate transport class.
Args:
label: The name of the desired transport. If none is
provided, then the first transport in the registry is used.
Returns:
The transport class to use.
"""
# If a specific transport is requested, return that one.
if label:
return cls._transport_registry[label]
# No transport is requested; return the default (that is, the first one
# in the dictionary).
return next(iter(cls._transport_registry.values()))
class CampaignSimulationServiceClient(
metaclass=CampaignSimulationServiceClientMeta
):
"""Service to fetch campaign simulations."""
@staticmethod
def _get_default_mtls_endpoint(api_endpoint):
"""Convert api endpoint to mTLS endpoint.
Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to
"*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively.
Args:
api_endpoint (Optional[str]): the api endpoint to convert.
Returns:
str: converted mTLS api endpoint.
"""
if not api_endpoint:
return api_endpoint
mtls_endpoint_re = re.compile(
r"(?P<name>[^.]+)(?P<mtls>\.mtls)?(?P<sandbox>\.sandbox)?(?P<googledomain>\.googleapis\.com)?"
)
m = mtls_endpoint_re.match(api_endpoint)
name, mtls, sandbox, googledomain = m.groups()
if mtls or not googledomain:
return api_endpoint
if sandbox:
return api_endpoint.replace(
"sandbox.googleapis.com", "mtls.sandbox.googleapis.com"
)
return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com")
DEFAULT_ENDPOINT = "googleads.googleapis.com"
DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore
DEFAULT_ENDPOINT
)
@classmethod
def from_service_account_info(cls, info: dict, *args, **kwargs):
"""Creates an instance of this client using the provided credentials info.
Args:
info (dict): The service account private key info.
args: Additional arguments to pass to the constructor.
kwargs: Additional arguments to pass to the constructor.
Returns:
CampaignSimulationServiceClient: The constructed client.
"""
credentials = service_account.Credentials.from_service_account_info(
info
)
kwargs["credentials"] = credentials
return cls(*args, **kwargs)
@classmethod
def from_service_account_file(cls, filename: str, *args, **kwargs):
"""Creates an instance of this client using the provided credentials
file.
Args:
filename (str): The path to the service account private key json
file.
args: Additional arguments to pass to the constructor.
kwargs: Additional arguments to pass to the constructor.
Returns:
CampaignSimulationServiceClient: The constructed client.
"""
credentials = service_account.Credentials.from_service_account_file(
filename
)
kwargs["credentials"] = credentials
return cls(*args, **kwargs)
from_service_account_json = from_service_account_file
@property
def transport(self) -> CampaignSimulationServiceTransport:
"""Return the transport used by the client instance.
Returns:
CampaignSimulationServiceTransport: The transport used by the client instance.
"""
return self._transport
@staticmethod
def campaign_simulation_path(
customer_id: str,
campaign_id: str,
type: str,
modification_method: str,
start_date: str,
end_date: str,
) -> str:
"""Return a fully-qualified campaign_simulation string."""
return "customers/{customer_id}/campaignSimulations/{campaign_id}~{type}~{modification_method}~{start_date}~{end_date}".format(
customer_id=customer_id,
campaign_id=campaign_id,
type=type,
modification_method=modification_method,
start_date=start_date,
end_date=end_date,
)
@staticmethod
def parse_campaign_simulation_path(path: str) -> Dict[str, str]:
"""Parse a campaign_simulation path into its component segments."""
m = re.match(
r"^customers/(?P<customer_id>.+?)/campaignSimulations/(?P<campaign_id>.+?)~(?P<type>.+?)~(?P<modification_method>.+?)~(?P<start_date>.+?)~(?P<end_date>.+?)$",
path,
)
return m.groupdict() if m else {}
@staticmethod
def common_billing_account_path(billing_account: str,) -> str:
"""Return a fully-qualified billing_account string."""
return "billingAccounts/{billing_account}".format(
billing_account=billing_account,
)
@staticmethod
def parse_common_billing_account_path(path: str) -> Dict[str, str]:
"""Parse a billing_account path into its component segments."""
m = re.match(r"^billingAccounts/(?P<billing_account>.+?)$", path)
return m.groupdict() if m else {}
@staticmethod
def common_folder_path(folder: str,) -> str:
"""Return a fully-qualified folder string."""
return "folders/{folder}".format(folder=folder,)
@staticmethod
def parse_common_folder_path(path: str) -> Dict[str, str]:
"""Parse a folder path into its component segments."""
m = re.match(r"^folders/(?P<folder>.+?)$", path)
return m.groupdict() if m else {}
@staticmethod
def common_organization_path(organization: str,) -> str:
"""Return a fully-qualified organization string."""
return "organizations/{organization}".format(organization=organization,)
@staticmethod
def parse_common_organization_path(path: str) -> Dict[str, str]:
"""Parse a organization path into its component segments."""
m = re.match(r"^organizations/(?P<organization>.+?)$", path)
return m.groupdict() if m else {}
@staticmethod
def common_project_path(project: str,) -> str:
"""Return a fully-qualified project string."""
return "projects/{project}".format(project=project,)
@staticmethod
def parse_common_project_path(path: str) -> Dict[str, str]:
"""Parse a project path into its component segments."""
m = re.match(r"^projects/(?P<project>.+?)$", path)
return m.groupdict() if m else {}
@staticmethod
def common_location_path(project: str, location: str,) -> str:
"""Return a fully-qualified location string."""
return "projects/{project}/locations/{location}".format(
project=project, location=location,
)
@staticmethod
def parse_common_location_path(path: str) -> Dict[str, str]:
"""Parse a location path into its component segments."""
m = re.match(
r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)$", path
)
return m.groupdict() if m else {}
def __init__(
self,
*,
credentials: Optional[ga_credentials.Credentials] = None,
transport: Union[str, CampaignSimulationServiceTransport, None] = None,
client_options: Optional[client_options_lib.ClientOptions] = None,
client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO,
) -> None:
"""Instantiate the campaign simulation service client.
Args:
credentials (Optional[google.auth.credentials.Credentials]): The
authorization credentials to attach to requests. These
credentials identify the application to the service; if none
are specified, the client will attempt to ascertain the
credentials from the environment.
transport (Union[str, ~.CampaignSimulationServiceTransport]): The
transport to use. If set to None, a transport is chosen
automatically.
client_options (google.api_core.client_options.ClientOptions): Custom options for the
client. It won't take effect if a ``transport`` instance is provided.
(1) The ``api_endpoint`` property can be used to override the
default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT
environment variable can also be used to override the endpoint:
"always" (always use the default mTLS endpoint), "never" (always
use the default regular endpoint) and "auto" (auto switch to the
default mTLS endpoint if client certificate is present, this is
the default value). However, the ``api_endpoint`` property takes
precedence if provided.
(2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable
is "true", then the ``client_cert_source`` property can be used
to provide client certificate for mutual TLS transport. If
not provided, the default SSL client certificate will be used if
present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not
set, no client certificate will be used.
client_info (google.api_core.gapic_v1.client_info.ClientInfo):
The client info used to send a user-agent string along with
API requests. If ``None``, then default info will be used.
Generally, you only need to set this if you're developing
your own client library.
Raises:
google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport
creation failed for any reason.
"""
if isinstance(client_options, dict):
client_options = client_options_lib.from_dict(client_options)
if client_options is None:
client_options = client_options_lib.ClientOptions()
# Create SSL credentials for mutual TLS if needed.
use_client_cert = bool(
util.strtobool(
os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false")
)
)
ssl_credentials = None
is_mtls = False
if use_client_cert:
if client_options.client_cert_source:
import grpc # type: ignore
cert, key = client_options.client_cert_source()
ssl_credentials = grpc.ssl_channel_credentials(
certificate_chain=cert, private_key=key
)
is_mtls = True
else:
creds = SslCredentials()
is_mtls = creds.is_mtls
ssl_credentials = creds.ssl_credentials if is_mtls else None
# Figure out which api endpoint to use.
if client_options.api_endpoint is not None:
api_endpoint = client_options.api_endpoint
else:
use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto")
if use_mtls_env == "never":
api_endpoint = self.DEFAULT_ENDPOINT
elif use_mtls_env == "always":
api_endpoint = self.DEFAULT_MTLS_ENDPOINT
elif use_mtls_env == "auto":
api_endpoint = (
self.DEFAULT_MTLS_ENDPOINT
if is_mtls
else self.DEFAULT_ENDPOINT
)
else:
raise MutualTLSChannelError(
"Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted values: never, auto, always"
)
# Save or instantiate the transport.
# Ordinarily, we provide the transport, but allowing a custom transport
# instance provides an extensibility point for unusual situations.
if isinstance(transport, CampaignSimulationServiceTransport):
# transport is a CampaignSimulationServiceTransport instance.
if credentials:
raise ValueError(
"When providing a transport instance, "
"provide its credentials directly."
)
self._transport = transport
elif isinstance(transport, str):
Transport = type(self).get_transport_class(transport)
self._transport = Transport(
credentials=credentials, host=self.DEFAULT_ENDPOINT
)
else:
self._transport = CampaignSimulationServiceGrpcTransport(
credentials=credentials,
host=api_endpoint,
ssl_channel_credentials=ssl_credentials,
client_info=client_info,
)
def get_campaign_simulation(
self,
request: campaign_simulation_service.GetCampaignSimulationRequest = None,
*,
resource_name: str = None,
retry: retries.Retry = gapic_v1.method.DEFAULT,
timeout: float = None,
metadata: Sequence[Tuple[str, str]] = (),
) -> campaign_simulation.CampaignSimulation:
r"""Returns the requested campaign simulation in full
detail.
Args:
request (:class:`google.ads.googleads.v8.services.types.GetCampaignSimulationRequest`):
The request object. Request message for
[CampaignSimulationService.GetCampaignSimulation][google.ads.googleads.v8.services.CampaignSimulationService.GetCampaignSimulation].
resource_name (:class:`str`):
Required. The resource name of the
campaign simulation to fetch.
This corresponds to the ``resource_name`` field
on the ``request`` instance; if ``request`` is provided, this
should not be set.
retry (google.api_core.retry.Retry): Designation of what errors, if any,
should be retried.
timeout (float): The timeout for this request.
metadata (Sequence[Tuple[str, str]]): Strings which should be
sent along with the request as metadata.
Returns:
google.ads.googleads.v8.resources.types.CampaignSimulation:
A campaign simulation. Supported combinations of advertising
channel type, simulation type and simulation
modification method is detailed below respectively.
SEARCH - CPC_BID - UNIFORM SEARCH - CPC_BID - SCALING
SEARCH - TARGET_CPA - UNIFORM SEARCH - TARGET_CPA -
SCALING SEARCH - TARGET_ROAS - UNIFORM SEARCH -
TARGET_IMPRESSION_SHARE - UNIFORM SEARCH - BUDGET -
UNIFORM SHOPPING - BUDGET - UNIFORM SHOPPING -
TARGET_ROAS - UNIFORM MULTIPLE - TARGET_CPA - UNIFORM
OWNED_AND_OPERATED - TARGET_CPA - DEFAULT DISPLAY -
TARGET_CPA - UNIFORM
"""
# Create or coerce a protobuf request object.
# Sanity check: If we got a request object, we should *not* have
# gotten any keyword arguments that map to the request.
if request is not None and any([resource_name]):
raise ValueError(
"If the `request` argument is set, then none of "
"the individual field arguments should be set."
)
# Minor optimization to avoid making a copy if the user passes
# in a campaign_simulation_service.GetCampaignSimulationRequest.
# There's no risk of modifying the input as we've already verified
# there are no flattened fields.
if not isinstance(
request, campaign_simulation_service.GetCampaignSimulationRequest
):
request = campaign_simulation_service.GetCampaignSimulationRequest(
request
)
# If we have keyword arguments corresponding to fields on the
# request, apply these.
if resource_name is not None:
request.resource_name = resource_name
# Wrap the RPC method; this adds retry and timeout information,
# and friendly error handling.
rpc = self._transport._wrapped_methods[
self._transport.get_campaign_simulation
]
# Certain fields should be provided within the metadata header;
# add these here.
metadata = tuple(metadata) + (
gapic_v1.routing_header.to_grpc_metadata(
(("resource_name", request.resource_name),)
),
)
# Send the request.
response = rpc(
request, retry=retry, timeout=timeout, metadata=metadata,
)
# Done; return the response.
return response
__all__ = ("CampaignSimulationServiceClient",)
|
|
# -*- coding: utf-8 -*-
############################ Copyrights and license ############################
# #
# Copyright 2012 Vincent Jacques <vincent@vincent-jacques.net> #
# Copyright 2012 Zearin <zearin@gonk.net> #
# Copyright 2013 AKFish <akfish@gmail.com> #
# Copyright 2013 Vincent Jacques <vincent@vincent-jacques.net> #
# Copyright 2013 martinqt <m.ki2@laposte.net> #
# Copyright 2014 Andy Casey <acasey@mso.anu.edu.au> #
# Copyright 2014 Vincent Jacques <vincent@vincent-jacques.net> #
# Copyright 2016 Jannis Gebauer <ja.geb@me.com> #
# Copyright 2016 John Eskew <jeskew@edx.org> #
# Copyright 2016 Peter Buckley <dx-pbuckley@users.noreply.github.com> #
# Copyright 2018 sfdye <tsfdye@gmail.com> #
# #
# This file is part of PyGithub. #
# http://pygithub.readthedocs.io/ #
# #
# PyGithub is free software: you can redistribute it and/or modify it under #
# the terms of the GNU Lesser General Public License as published by the Free #
# Software Foundation, either version 3 of the License, or (at your option) #
# any later version. #
# #
# PyGithub is distributed in the hope that it will be useful, but WITHOUT ANY #
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS #
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more #
# details. #
# #
# You should have received a copy of the GNU Lesser General Public License #
# along with PyGithub. If not, see <http://www.gnu.org/licenses/>. #
# #
################################################################################
import github.GithubObject
import github.PaginatedList
import github.GitCommit
import github.NamedUser
import github.CommitStatus
import github.CommitCombinedStatus
import github.File
import github.CommitStats
import github.CommitComment
class Commit(github.GithubObject.CompletableGithubObject):
"""
This class represents Commits. The reference can be found here http://developer.github.com/v3/git/commits/
"""
def __repr__(self):
return self.get__repr__({"sha": self._sha.value})
@property
def author(self):
"""
:type: :class:`github.NamedUser.NamedUser`
"""
self._completeIfNotSet(self._author)
return self._author.value
@property
def comments_url(self):
"""
:type: string
"""
self._completeIfNotSet(self._comments_url)
return self._comments_url.value
@property
def commit(self):
"""
:type: :class:`github.GitCommit.GitCommit`
"""
self._completeIfNotSet(self._commit)
return self._commit.value
@property
def committer(self):
"""
:type: :class:`github.NamedUser.NamedUser`
"""
self._completeIfNotSet(self._committer)
return self._committer.value
@property
def files(self):
"""
:type: list of :class:`github.File.File`
"""
self._completeIfNotSet(self._files)
return self._files.value
@property
def html_url(self):
"""
:type: string
"""
self._completeIfNotSet(self._html_url)
return self._html_url.value
@property
def parents(self):
"""
:type: list of :class:`github.Commit.Commit`
"""
self._completeIfNotSet(self._parents)
return self._parents.value
@property
def sha(self):
"""
:type: string
"""
self._completeIfNotSet(self._sha)
return self._sha.value
@property
def stats(self):
"""
:type: :class:`github.CommitStats.CommitStats`
"""
self._completeIfNotSet(self._stats)
return self._stats.value
@property
def url(self):
"""
:type: string
"""
self._completeIfNotSet(self._url)
return self._url.value
def create_comment(self, body, line=github.GithubObject.NotSet, path=github.GithubObject.NotSet, position=github.GithubObject.NotSet):
"""
:calls: `POST /repos/:owner/:repo/commits/:sha/comments <http://developer.github.com/v3/repos/comments>`_
:param body: string
:param line: integer
:param path: string
:param position: integer
:rtype: :class:`github.CommitComment.CommitComment`
"""
assert isinstance(body, (str, unicode)), body
assert line is github.GithubObject.NotSet or isinstance(line, (int, long)), line
assert path is github.GithubObject.NotSet or isinstance(path, (str, unicode)), path
assert position is github.GithubObject.NotSet or isinstance(position, (int, long)), position
post_parameters = {
"body": body,
}
if line is not github.GithubObject.NotSet:
post_parameters["line"] = line
if path is not github.GithubObject.NotSet:
post_parameters["path"] = path
if position is not github.GithubObject.NotSet:
post_parameters["position"] = position
headers, data = self._requester.requestJsonAndCheck(
"POST",
self.url + "/comments",
input=post_parameters
)
return github.CommitComment.CommitComment(self._requester, headers, data, completed=True)
def create_status(self, state, target_url=github.GithubObject.NotSet, description=github.GithubObject.NotSet, context=github.GithubObject.NotSet):
"""
:calls: `POST /repos/:owner/:repo/statuses/:sha <http://developer.github.com/v3/repos/statuses>`_
:param state: string
:param target_url: string
:param description: string
:param context: string
:rtype: :class:`github.CommitStatus.CommitStatus`
"""
assert isinstance(state, (str, unicode)), state
assert target_url is github.GithubObject.NotSet or isinstance(target_url, (str, unicode)), target_url
assert description is github.GithubObject.NotSet or isinstance(description, (str, unicode)), description
assert context is github.GithubObject.NotSet or isinstance(context, (str, unicode)), context
post_parameters = {
"state": state,
}
if target_url is not github.GithubObject.NotSet:
post_parameters["target_url"] = target_url
if description is not github.GithubObject.NotSet:
post_parameters["description"] = description
if context is not github.GithubObject.NotSet:
post_parameters["context"] = context
headers, data = self._requester.requestJsonAndCheck(
"POST",
self._parentUrl(self._parentUrl(self.url)) + "/statuses/" + self.sha,
input=post_parameters
)
return github.CommitStatus.CommitStatus(self._requester, headers, data, completed=True)
def get_comments(self):
"""
:calls: `GET /repos/:owner/:repo/commits/:sha/comments <http://developer.github.com/v3/repos/comments>`_
:rtype: :class:`github.PaginatedList.PaginatedList` of :class:`github.CommitComment.CommitComment`
"""
return github.PaginatedList.PaginatedList(
github.CommitComment.CommitComment,
self._requester,
self.url + "/comments",
None
)
def get_statuses(self):
"""
:calls: `GET /repos/:owner/:repo/statuses/:ref <http://developer.github.com/v3/repos/statuses>`_
:rtype: :class:`github.PaginatedList.PaginatedList` of :class:`github.CommitStatus.CommitStatus`
"""
return github.PaginatedList.PaginatedList(
github.CommitStatus.CommitStatus,
self._requester,
self._parentUrl(self._parentUrl(self.url)) + "/statuses/" + self.sha,
None
)
def get_combined_status(self):
"""
:calls: `GET /repos/:owner/:repo/commits/:ref/status/ <http://developer.github.com/v3/repos/statuses>`_
:rtype: :class:`github.CommitCombinedStatus.CommitCombinedStatus`
"""
headers, data = self._requester.requestJsonAndCheck(
"GET",
self.url + "/status"
)
return github.CommitCombinedStatus.CommitCombinedStatus(self._requester, headers, data, completed=True)
@property
def _identity(self):
return self.sha
def _initAttributes(self):
self._author = github.GithubObject.NotSet
self._comments_url = github.GithubObject.NotSet
self._commit = github.GithubObject.NotSet
self._committer = github.GithubObject.NotSet
self._files = github.GithubObject.NotSet
self._html_url = github.GithubObject.NotSet
self._parents = github.GithubObject.NotSet
self._sha = github.GithubObject.NotSet
self._stats = github.GithubObject.NotSet
self._url = github.GithubObject.NotSet
def _useAttributes(self, attributes):
if "author" in attributes: # pragma no branch
self._author = self._makeClassAttribute(github.NamedUser.NamedUser, attributes["author"])
if "comments_url" in attributes: # pragma no branch
self._comments_url = self._makeStringAttribute(attributes["comments_url"])
if "commit" in attributes: # pragma no branch
self._commit = self._makeClassAttribute(github.GitCommit.GitCommit, attributes["commit"])
if "committer" in attributes: # pragma no branch
self._committer = self._makeClassAttribute(github.NamedUser.NamedUser, attributes["committer"])
if "files" in attributes: # pragma no branch
self._files = self._makeListOfClassesAttribute(github.File.File, attributes["files"])
if "html_url" in attributes: # pragma no branch
self._html_url = self._makeStringAttribute(attributes["html_url"])
if "parents" in attributes: # pragma no branch
self._parents = self._makeListOfClassesAttribute(Commit, attributes["parents"])
if "sha" in attributes: # pragma no branch
self._sha = self._makeStringAttribute(attributes["sha"])
if "stats" in attributes: # pragma no branch
self._stats = self._makeClassAttribute(github.CommitStats.CommitStats, attributes["stats"])
if "url" in attributes: # pragma no branch
self._url = self._makeStringAttribute(attributes["url"])
|
|
# -*- coding: utf-8 -*-
#
# Copyright (c) 2020, the cclib development team
#
# This file is part of cclib (http://cclib.github.io) and is distributed under
# the terms of the BSD 3-Clause License.
"""Parser for MOPAC output files"""
# Based on parser in RMG-Py by Greg Magoon
# https://github.com/ReactionMechanismGenerator/RMG-Py/blob/master/external/cclib/parser/mopacparser.py
# Also parts from Ben Albrecht
# https://github.com/ben-albrecht/cclib/blob/master/cclib/parser/mopacparser.py
# Merged and modernized by Geoff Hutchison
import re
import math
import numpy
from cclib.parser import data
from cclib.parser import logfileparser
from cclib.parser import utils
def symbol2int(symbol):
t = utils.PeriodicTable()
return t.number[symbol]
class MOPAC(logfileparser.Logfile):
"""A MOPAC20XX output file."""
def __init__(self, *args, **kwargs):
super().__init__(logname="MOPAC", *args, **kwargs)
def __str__(self):
"""Return a string representation of the object."""
return "MOPAC log file %s" % (self.filename)
def __repr__(self):
"""Return a representation of the object."""
return 'MOPAC("%s")' % (self.filename)
def normalisesym(self, label):
"""MOPAC does not require normalizing symmetry labels."""
return label
def before_parsing(self):
#TODO
# Defaults
charge = 0
self.set_attribute('charge', charge)
mult = 1
self.set_attribute('mult', mult)
# Keep track of whether or not we're performing an
# (un)restricted calculation.
self.unrestricted = False
self.is_rohf = False
# Keep track of 1SCF vs. gopt since gopt is default
self.onescf = False
self.geomdone = False
# Compile the dashes-and-or-spaces-only regex.
self.re_dashes_and_spaces = re.compile(r'^[\s-]+$')
self.star = ' * '
self.stars = ' *******************************************************************************'
self.spinstate = {'SINGLET': 1,
'DOUBLET': 2,
'TRIPLET': 3,
'QUARTET': 4,
'QUINTET': 5,
'SEXTET': 6,
'HEPTET': 7,
'OCTET': 8,
'NONET': 9}
def extract(self, inputfile, line):
"""Extract information from the file object inputfile."""
# Extract the package version.
if "For non-commercial use only" in line:
# Ignore the platorm information for now (the last character).
self.metadata["package_version"] = line.split()[8][:-1]
# Use the year as the legacy (short) package version.
self.skip_lines(
inputfile, ["Stewart Computational Chemistry", "s", "s", "s", "s"]
)
self.metadata["legacy_package_version"] = next(inputfile).split()[1][5:]
# Extract the atomic numbers and coordinates from the optimized geometry
# note that cartesian coordinates section occurs multiple times in the file, and we want to end up using the last instance
# also, note that the section labeled cartesian coordinates doesn't have as many decimal places as the one used here
# Example 1 (not used):
# CARTESIAN COORDINATES
#
# NO. ATOM X Y Z
#
# 1 O 4.7928 -0.8461 0.3641
# 2 O 5.8977 -0.3171 0.0092
# ...
# Example 2 (used):
# ATOM CHEMICAL X Y Z
# NUMBER SYMBOL (ANGSTROMS) (ANGSTROMS) (ANGSTROMS)
#
# 1 O 4.79280259 * -0.84610232 * 0.36409474 *
# 2 O 5.89768035 * -0.31706418 * 0.00917035 *
# ... etc.
if line.split() == ["NUMBER", "SYMBOL", "(ANGSTROMS)", "(ANGSTROMS)", "(ANGSTROMS)"]:
self.updateprogress(inputfile, "Attributes", self.cupdate)
self.inputcoords = []
self.inputatoms = []
blankline = inputfile.next()
atomcoords = []
line = inputfile.next()
while len(line.split()) > 6:
# MOPAC Version 14.019L 64BITS suddenly appends this block with
# "CARTESIAN COORDINATES" block with no blank line.
tokens = line.split()
self.inputatoms.append(symbol2int(tokens[1]))
xc = float(tokens[2])
yc = float(tokens[4])
zc = float(tokens[6])
atomcoords.append([xc, yc, zc])
line = inputfile.next()
self.inputcoords.append(atomcoords)
if not hasattr(self, "natom"):
self.atomnos = numpy.array(self.inputatoms, 'i')
self.natom = len(self.atomnos)
if 'CHARGE ON SYSTEM =' in line:
charge = int(line.split()[5])
self.set_attribute('charge', charge)
if 'SPIN STATE DEFINED' in line:
# find the multiplicity from the line token (SINGLET, DOUBLET, TRIPLET, etc)
mult = self.spinstate[line.split()[1]]
self.set_attribute('mult', mult)
# Read energy (in kcal/mol, converted to eV)
#
# FINAL HEAT OF FORMATION = -333.88606 KCAL = -1396.97927 KJ
if 'FINAL HEAT OF FORMATION =' in line:
if not hasattr(self, "scfenergies"):
self.scfenergies = []
self.scfenergies.append(utils.convertor(utils.float(line.split()[5]), "kcal/mol", "eV"))
# Molecular mass parsing (units will be amu)
#
# MOLECULAR WEIGHT == 130.1890
if line[0:35] == ' MOLECULAR WEIGHT =':
self.molmass = utils.float(line.split()[3])
#rotational constants
#Example:
# ROTATIONAL CONSTANTS IN CM(-1)
#
# A = 0.01757641 B = 0.00739763 C = 0.00712013
# could also read in moment of inertia, but this should just differ by a constant: rot cons= h/(8*Pi^2*I)
# note that the last occurence of this in the thermochemistry section has reduced precision,
# so we will want to use the 2nd to last instance
if line[0:40] == ' ROTATIONAL CONSTANTS IN CM(-1)':
blankline = inputfile.next()
rotinfo = inputfile.next()
if not hasattr(self, "rotcons"):
self.rotcons = []
broken = rotinfo.split()
# leave the rotational constants in Hz
a = float(broken[2])
b = float(broken[5])
c = float(broken[8])
self.rotcons.append([a, b, c])
# Start of the IR/Raman frequency section.
# Example:
# VIBRATION 1 1A ATOM PAIR ENERGY CONTRIBUTION RADIAL
# FREQ. 15.08 C 12 -- C 16 +7.9% (999.0%) 0.0%
# T-DIPOLE 0.2028 C 16 -- H 34 +5.8% (999.0%) 28.0%
# TRAVEL 0.0240 C 16 -- H 32 +5.6% (999.0%) 35.0%
# RED. MASS 1.7712 O 1 -- O 4 +5.2% (999.0%) 0.4%
# EFF. MASS7752.8338
#
# VIBRATION 2 2A ATOM PAIR ENERGY CONTRIBUTION RADIAL
# FREQ. 42.22 C 11 -- C 15 +9.0% (985.8%) 0.0%
# T-DIPOLE 0.1675 C 15 -- H 31 +6.6% (843.6%) 3.3%
# TRAVEL 0.0359 C 15 -- H 29 +6.0% (802.8%) 24.5%
# RED. MASS 1.7417 C 13 -- C 17 +5.8% (792.7%) 0.0%
# EFF. MASS1242.2114
if line[1:10] == 'VIBRATION':
self.updateprogress(inputfile, "Frequency Information", self.fupdate)
# get the vib symmetry
if len(line.split()) >= 3:
sym = line.split()[2]
if not hasattr(self, 'vibsyms'):
self.vibsyms = []
self.vibsyms.append(sym)
line = inputfile.next()
if 'FREQ' in line:
if not hasattr(self, 'vibfreqs'):
self.vibfreqs = []
freq = float(line.split()[1])
self.vibfreqs.append(freq)
line = inputfile.next()
if 'T-DIPOLE' in line:
if not hasattr(self, 'vibirs'):
self.vibirs = []
tdipole = float(line.split()[1])
# transform to km/mol
self.vibirs.append(math.sqrt(tdipole))
line = inputfile.next()
if 'TRAVEL' in line:
pass
line = inputfile.next()
if 'RED. MASS' in line:
if not hasattr(self, 'vibrmasses'):
self.vibrmasses = []
rmass = float(line.split()[2])
self.vibrmasses.append(rmass)
# Orbital eigenvalues, e.g.
# ALPHA EIGENVALUES
# BETA EIGENVALUES
# or just "EIGENVALUES" for closed-shell
if 'EIGENVALUES' in line:
if not hasattr(self, 'moenergies'):
self.moenergies = [] # list of arrays
energies = []
line = inputfile.next()
while len(line.split()) > 0:
energies.extend([float(i) for i in line.split()])
line = inputfile.next()
self.moenergies.append(energies)
# todo:
# Partial charges and dipole moments
# Example:
# NET ATOMIC CHARGES
if line[:16] == '== MOPAC DONE ==':
self.metadata['success'] = True
|
|
from __future__ import print_function, unicode_literals
import re
from decimal import Decimal as D
from aspen import Response
import pytest
from gratipay.security.user import SESSION
from gratipay.testing import Harness
from gratipay.wireup import find_files
overescaping_re = re.compile(r'&(#[0-9]{4}|[a-z]+);')
class TestPages(Harness):
def browse(self, setup=None, **kw):
alice = self.make_participant('alice', claimed_time='now', number='plural')
exchange_id = self.make_exchange('balanced-cc', 19, 0, alice)
alice.insert_into_communities(True, 'Wonderland', 'wonderland')
alan = self.make_participant('alan', claimed_time='now')
alice.add_member(alan)
if setup:
setup(alice)
i = len(self.client.www_root)
urls = []
for spt in find_files(self.client.www_root, '*.spt'):
url = spt[i:-4].replace('/%team/', '/alice/') \
.replace('/alice/%sub', '/alice/foo') \
.replace('/~/%username/', '/~alice/') \
.replace('/for/%slug/', '/for/wonderland/') \
.replace('/%platform/', '/github/') \
.replace('/%user_name/', '/gratipay/') \
.replace('/%membername', '/alan') \
.replace('/%exchange_id.int', '/%s' % exchange_id) \
.replace('/%redirect_to', '/giving') \
.replace('/%endpoint', '/public') \
.replace('/about/me/%sub', '/about/me')
assert '/%' not in url
if 'index' in url.split('/')[-1]:
url = url.rsplit('/', 1)[0] + '/'
urls.append(url)
urls.extend("""
/about/me
/about/me/
/about/me/history
""".split())
for url in urls:
try:
r = self.client.GET(url, **kw)
except Response as r:
if r.code == 404 or r.code >= 500:
raise
assert r.code != 404
assert r.code < 500
assert not overescaping_re.search(r.body.decode('utf8'))
def test_anon_can_browse(self):
self.browse()
def test_new_participant_can_browse(self):
self.browse(auth_as='alice')
def test_on_the_fence_can_browse(self):
def setup(alice):
bob = self.make_participant('bob', claimed_time='now', last_bill_result='')
bob.set_tip_to(alice, D('1.00'))
self.browse(setup, auth_as='alice')
def test_escaping_on_homepage(self):
self.make_participant('alice', claimed_time='now')
expected = "<a href='/alice/'>"
actual = self.client.GET('/', auth_as='alice').body
assert expected in actual
@pytest.mark.xfail(reason="migrating to Teams; #3399")
def test_username_is_in_button(self):
self.make_participant('alice', claimed_time='now')
self.make_participant('bob', claimed_time='now')
body = self.client.GET('/~alice/', auth_as='bob').body
assert '<span class="zero">Give to alice</span>' in body
@pytest.mark.xfail(reason="migrating to Teams; #3399")
def test_username_is_in_unauth_giving_cta(self):
self.make_participant('alice', claimed_time='now')
body = self.client.GET('/~alice/').body
assert 'give to alice' in body
def test_widget(self):
self.make_participant('cheese', claimed_time='now')
expected = "javascript: window.open"
actual = self.client.GET('/~cheese/widget.html').body
assert expected in actual
def test_github_associate(self):
assert self.client.GxT('/on/github/associate').code == 400
def test_twitter_associate(self):
assert self.client.GxT('/on/twitter/associate').code == 400
def test_about(self):
expected = "give money every week"
actual = self.client.GET('/about/').body
assert expected in actual
def test_about_stats(self):
expected = "have joined Gratipay"
actual = self.client.GET('/about/stats.html').body
assert expected in actual
def test_about_charts(self):
assert self.client.GxT('/about/charts.html').code == 302
def test_about_faq(self):
expected = "What is Gratipay?"
actual = self.client.GET('/about/faq.html').body.decode('utf8')
assert expected in actual
def test_about_teams_redirect(self):
assert self.client.GxT('/about/teams/').code == 302
def test_about_teams(self):
expected = "Teams"
actual = self.client.GET('/about/features/teams/').body.decode('utf8')
assert expected in actual
def test_404(self):
response = self.client.GET('/about/four-oh-four.html', raise_immediately=False)
assert "Not Found" in response.body
assert "{%" not in response.body
def test_for_contributors_redirects_to_inside_gratipay(self):
loc = self.client.GxT('/for/contributors/').headers['Location']
assert loc == 'http://inside.gratipay.com/'
def test_mission_statement_also_redirects(self):
assert self.client.GxT('/for/contributors/mission-statement.html').code == 302
def test_anonymous_sign_out_redirects(self):
response = self.client.PxST('/sign-out.html')
assert response.code == 302
assert response.headers['Location'] == '/'
def test_sign_out_overwrites_session_cookie(self):
self.make_participant('alice')
response = self.client.PxST('/sign-out.html', auth_as='alice')
assert response.code == 302
assert response.headers.cookie[SESSION].value == ''
def test_sign_out_doesnt_redirect_xhr(self):
self.make_participant('alice')
response = self.client.PxST('/sign-out.html', auth_as='alice',
HTTP_X_REQUESTED_WITH=b'XMLHttpRequest')
assert response.code == 200
def test_settings_page_available_balance(self):
self.make_participant('alice', claimed_time='now')
self.db.run("UPDATE participants SET balance = 123.00 WHERE username = 'alice'")
actual = self.client.GET("/~alice/settings/", auth_as="alice").body
expected = "123"
assert expected in actual
def test_subscriptions_page(self):
self.make_team(is_approved=True)
alice = self.make_participant('alice', claimed_time='now')
alice.set_subscription_to('TheATeam', "1.00")
assert "The A Team" in self.client.GET("/~alice/subscriptions/", auth_as="alice").body
def test_giving_page_shows_cancelled(self):
self.make_team(is_approved=True)
alice = self.make_participant('alice', claimed_time='now')
alice.set_subscription_to('TheATeam', "1.00")
alice.set_subscription_to('TheATeam', "0.00")
assert "Cancelled" in self.client.GET("/~alice/subscriptions/", auth_as="alice").body
def test_new_participant_can_edit_profile(self):
self.make_participant('alice', claimed_time='now')
body = self.client.GET("/~alice/", auth_as="alice").body
assert b'Edit' in body
def test_tilde_slash_redirects_to_tilde(self):
self.make_participant('alice', claimed_time='now')
response = self.client.GxT("/~/alice/", auth_as="alice")
assert response.code == 302
assert response.headers['Location'] == '/~alice/'
def test_tilde_slash_redirects_subpages_with_querystring_to_tilde(self):
self.make_participant('alice', claimed_time='now')
response = self.client.GxT("/~/alice/foo/bar?baz=buz", auth_as="alice")
assert response.code == 302
assert response.headers['Location'] == '/~alice/foo/bar?baz=buz'
def test_username_redirected_to_tilde(self):
self.make_participant('alice', claimed_time='now')
response = self.client.GxT("/alice/", auth_as="alice")
assert response.code == 302
assert response.headers['Location'] == '/~alice/'
def test_username_redirects_everything_to_tilde(self):
self.make_participant('alice', claimed_time='now')
response = self.client.GxT("/alice/foo/bar?baz=buz", auth_as="alice")
assert response.code == 302
assert response.headers['Location'] == '/~alice/foo/bar?baz=buz'
def test_team_slug__not__redirected_from_tilde(self):
self.make_team(is_approved=True)
assert self.client.GET("/TheATeam/").code == 200
assert self.client.GxT("/~TheATeam/").code == 404
|
|
from django.contrib import admin
from django.contrib.admin import SimpleListFilter
from django.db import models
from django.db.models.fields import CharField, TextField
from django.forms import Textarea, ModelForm
from import_export.admin import ImportExportModelAdmin
from solo.admin import SingletonModelAdmin
from .models import Audiologist
from .models import AudiologistResource
from .models import Client
from .models import ClientResource
from .models import MeetingLog
from .models import MeetingLogResource
from .models import Provider
from .models import ProviderResource
from .models import IncomeSource
from .models import Settings
from .models import Grantor
from .models import GrantorResource
standard_textarea = Textarea(attrs={'rows': 3,
'cols': 40,
'style': 'height: 3.6em;'})
class DeleteNotAllowedModelAdmin(admin.ModelAdmin):
def has_delete_permission(self, request, obj=None):
return request.user.is_superuser
class AudiologistCurrentFilter(SimpleListFilter):
'''
Custom filter that defaults to "current" == True
'''
title = 'Status'
parameter_name = 'current'
def lookups(self, request, model_admin):
return (
('a', 'All audiologists'),
('y', 'Current'),
('n', 'Inactive'),
)
def queryset(self, request, queryset):
if self.value() == 'a':
return queryset.filter()
url_val_map = {
'y': True,
'n': False,
None: True,
}
val = url_val_map[self.value()]
return queryset.filter(current=val)
def choices(self, cl, *a, **kw):
yield {
'selected': self.value() is None or self.value == 'y',
'query_string': cl.get_query_string({}, [self.parameter_name]),
'display': 'Current',
}
yield {
'selected': self.value() == 'n',
'query_string': cl.get_query_string({self.parameter_name: 'n'}, []),
'display': 'Inactive',
}
yield {
'selected': self.value() == 'a',
'query_string': cl.get_query_string({self.parameter_name: 'a'}, []),
'display': 'All',
}
class AudiologistAdmin(DeleteNotAllowedModelAdmin, ImportExportModelAdmin):
list_display = ('name', 'allowed', 'current')
list_filter = (AudiologistCurrentFilter,)
ordering = ('name',)
resource_class = AudiologistResource
formfield_overrides = {
models.TextField: {
'widget': standard_textarea,
},
}
class ClientIncomeInlineAdmin(admin.TabularInline):
model = IncomeSource
can_delete = True
extra = 1
class MeetingLogInlineAdminForm(ModelForm):
class Meta:
model = MeetingLog
fields = '__all__'
widgets = {
'results': standard_textarea,
}
class MeetingLogInlineAdmin(admin.TabularInline):
model = MeetingLog
form = MeetingLogInlineAdminForm
can_delete = True
extra = 1
class DateYesNoFilter(SimpleListFilter):
def lookups(self, request, model_admin):
return (
('y', 'Yes'),
('n', 'No'),
)
def queryset(self, request, queryset):
query = {}
if self.value() == 'y':
query = {self.field_name + '__isnull': False}
elif self.value() == 'n':
query = {self.field_name + '__isnull': True}
return queryset.filter(**query)
class DeceasedFilter(DateYesNoFilter):
title = 'Deceased'
parameter_name = 'deceased'
field_name = 'date_of_death'
class CostShareApprovedFilter(DateYesNoFilter):
title = 'Cost Share Approved'
parameter_name = 'cost_share_approved'
field_name = 'cost_share_approval'
class UpdateMeetingFilter(DateYesNoFilter):
title = 'Had Update Meeting'
parameter_name = 'update_meeting'
field_name = 'update_meeting'
class ProviderAuthReqFilter(DateYesNoFilter):
title = 'Provider Auth Requested'
parameter_name = 'provider_auth_requested'
field_name = 'provider_auth_requested'
class ProviderAuthRecvFilter(DateYesNoFilter):
title = 'Provider Auth Rcvd'
parameter_name = 'provider_auth_received'
field_name = 'provider_auth_received'
class AudiologistReferredFilter(DateYesNoFilter):
title = 'Audiologist Referred'
parameter_name = 'audiologist_referral_date'
field_name = 'audiologist_referral_date'
class AudiologistApptFilter(DateYesNoFilter):
title = 'Audiologist Appt Set'
parameter_name = 'audiologist_appointment_date'
field_name = 'audiologist_appointment_date'
class AudiologistInvoicedFilter(DateYesNoFilter):
title = 'Audiologist Invoiced'
parameter_name = 'audiologist_invoiced_date'
field_name = 'audiologist_invoiced_date'
class ClientAdmin(ImportExportModelAdmin):
resource_class = ClientResource
list_display = ('last_name', 'first_name', 'intake_date', 'last_updated', 'hearing_loss', 'audiologist', 'client_grantors', 'cost_share', 'cost_share_approval')
list_display_links = ('last_name', 'first_name',)
list_filter = ('provider', 'audiologist', 'grantors', 'family_size', 'hearing_loss',
DeceasedFilter, CostShareApprovedFilter, UpdateMeetingFilter, 'update_meeting',
ProviderAuthReqFilter, ProviderAuthRecvFilter,
AudiologistReferredFilter, AudiologistApptFilter, AudiologistInvoicedFilter,
'equipment_requested', 'adaptive_equipment', 'hearing_aid_assistance',
'last_updated',
'quota_client', 'deliverable', 'non_kcsm',
'intake_staff', 'data_entry_staff')
ordering = ('-intake_date',)
date_hierarchy = 'intake_date'
search_fields = [f.name for f in Client._meta.local_fields if isinstance(f, (CharField, TextField))]
formfield_overrides = {
models.TextField: {
'widget': standard_textarea,
},
}
inlines = (ClientIncomeInlineAdmin,MeetingLogInlineAdmin)
readonly_fields = ('id', 'last_updated')
fieldsets = (
(None, {
'fields': (
'id', 'napis_id',
)
}),
('Personal Info', {
'fields': (
'first_name', 'last_name', 'gender', 'date_of_birth', 'date_of_death',
'is_veteran', 'lives_alone', 'spouse', 'family_size',
)
}),
('Contact', {
'fields': (
'address', 'city', 'county', 'state', 'zip_code', 'deliverable',
'email', 'phone',
'emergency_contact',
'emergency_phone',
)
}),
('Notes', {
'fields': (
'notes',
)
}),
('Demographics', {
'fields': (
'race', 'is_hispanic',
'multiracial', 'multiracial_white', 'multiracial_black', 'multiracial_asian', 'multiracial_amind',
)
}),
('Assistance', {
'fields': (
'hearing_loss', 'aids_requested_left', 'aids_requested_right', 'equipment_requested',
'hearing_assistance', 'adaptive_equipment', 'hearing_aid_assistance',
'equipment_borrowed',
)
}),
('Additional Forms', {
'fields': (
'proof_of_age', 'signed_client_intake', 'signed_disclosure_authorization',
'signed_confidentiality_policy', 'signed_gross_annual_income',
'signed_client_responsibility_fees'
)
}),
('DHHS', {
'fields': (
'intake_date', 'intake_staff', 'data_entry_staff', 'last_updated', 'referrer',
'update_meeting',
'cost_share_approval', 'cost_share',
'quota_client', 'non_kcsm', 'grantors',
'provider', 'audient_id', 'provider_auth_requested', 'provider_auth_received',
)
}),
('Audiologist', {
'fields': (
'audiologist', 'audiologist_referral_date', 'audiologist_appointment_date',
'audiologist_invoiced_date', 'audiologist_invoiced_amount',
)
}),
)
class MeetingLogAdmin(ImportExportModelAdmin):
resource_class = MeetingLogResource
list_display = ('client', 'contact_date', 'consultation_time', 'paperwork_time', 'units', 'results', 'user')
list_display_links = ('contact_date',)
list_filter = ('client', 'contact_date', 'user')
ordering = ('-contact_date',)
date_hierarchy = 'contact_date'
formfield_overrides = {
models.TextField: {
'widget': standard_textarea,
},
}
def units(self, obj):
return (obj.consultation_time + obj.paperwork_time) / 60
class ProviderAdmin(ImportExportModelAdmin):
ordering = ('name',)
resource_class = ProviderResource
formfield_overrides = {
models.TextField: {
'widget': standard_textarea,
},
}
class GrantorAdmin(ImportExportModelAdmin):
ordering = ('name',)
resource_class = GrantorResource
formfield_overrides = {
models.TextField: {
'widget': standard_textarea,
},
}
admin.site.disable_action('delete_selected')
admin.site.site_header = 'Deaf & Hard of Hearing Services - ADAPT'
admin.site.site_title = 'ADAPT'
admin.site.site_url = None
admin.site.index_title = ''
admin.site.register(Audiologist, AudiologistAdmin)
admin.site.register(Client, ClientAdmin)
admin.site.register(Provider, ProviderAdmin)
admin.site.register(Grantor, GrantorAdmin)
admin.site.register(MeetingLog, MeetingLogAdmin)
admin.site.register(Settings, SingletonModelAdmin)
|
|
"""
Tests for both experiment.py and experiment_set.py
"""
import pytest
from snovault import TYPES
# from snovault.storage import UUID
from uuid import uuid4
from ..types.experiment import ExperimentHiC
pytestmark = [pytest.mark.setone, pytest.mark.working]
@pytest.fixture
def custom_experiment_set_data(lab, award):
return {
'lab': lab['@id'],
'award': award['@id'],
'description': 'test experiment set',
'experimentset_type': 'custom',
'status': 'in review by lab'
}
@pytest.fixture
def custom_experiment_set(testapp, custom_experiment_set_data):
return testapp.post_json('/experiment_set', custom_experiment_set_data).json['@graph'][0]
@pytest.fixture
def replicate_experiment_set_data(lab, award):
return {
'lab': lab['@id'],
'award': award['@id'],
'description': 'test replicate set',
'experimentset_type': 'replicate',
'status': 'in review by lab'
}
@pytest.fixture
def replicate_experiment_set(testapp, replicate_experiment_set_data):
return testapp.post_json('/experiment_set_replicate', replicate_experiment_set_data).json['@graph'][0]
@pytest.fixture
def sop_map_data(protocol, lab, award):
return {
"sop_name": "in situ Hi-C SOP map",
"sop_version": 1,
'lab': lab['@id'],
'award': award['@id'],
"associated_item_type": "ExperimentHiC",
"id_values": ["in situ Hi-C"],
"notes": "This is just a dummy insert not linked to true SOP protocol",
"description": "Fields with specified defaults in the SOP for in situ Hi-C experiments as per ??",
"sop_protocol": protocol['@id'],
"fields_with_default": [
{"field_name": "digestion_enzyme", "field_value": "MboI"},
]
}
@pytest.fixture
def sop_map_data_2(lab, award):
return {
"sop_name": "Second in situ hic map",
"sop_version": 2,
'lab': lab['@id'],
'award': award['@id'],
"associated_item_type": "ExperimentHiC",
"id_values": ["in situ Hi-C"],
"notes": "This is a dummy second version of map",
"description": "Second",
}
def test_experiment_update_experiment_relation(testapp, base_experiment, experiment):
relation = [{'relationship_type': 'controlled by',
'experiment': experiment['@id']}]
res = testapp.patch_json(base_experiment['@id'], {'experiment_relation': relation})
assert res.json['@graph'][0]['experiment_relation'] == relation
# patching an experiement should also update the related experiement
exp_res = testapp.get(experiment['@id'])
exp_res_id = exp_res.json['experiment_relation'][0]['experiment']['@id']
assert exp_res_id == base_experiment['@id']
def test_experiment_update_hic_sop_mapping_added_on_submit(testapp, experiment_data, sop_map_data):
res_sop = testapp.post_json('/sop_map', sop_map_data, status=201)
res_exp = testapp.post_json('/experiment_hi_c', experiment_data)
assert 'sop_mapping' in res_exp.json['@graph'][0]
assert res_exp.json['@graph'][0]['sop_mapping']['has_sop'] == "Yes"
assert res_exp.json['@graph'][0]['sop_mapping']['sop_map'] == res_sop.json['@graph'][0]['@id']
def test_experiment_update_hic_sop_mapping_has_map_is_no(testapp, experiment_data, exp_types):
experiment_data['experiment_type'] = exp_types['dnase']['@id']
res_exp = testapp.post_json('/experiment_hi_c', experiment_data)
assert 'sop_mapping' in res_exp.json['@graph'][0]
assert res_exp.json['@graph'][0]['sop_mapping']['has_sop'] == "No"
def test_experiment_update_hic_sop_mapping_has_sop2no_when_only_sopmap_deleted(
testapp, experiment_data, sop_map_data):
sop_map_data['status'] = 'deleted'
testapp.post_json('/sop_map', sop_map_data, status=201)
res_exp = testapp.post_json('/experiment_hi_c', experiment_data)
assert 'sop_mapping' in res_exp.json['@graph'][0]
assert res_exp.json['@graph'][0]['sop_mapping']['has_sop'] == "No"
def test_experiment_update_hic_sop_mapping_to_v2_when_2_versions(
testapp, experiment_data, sop_map_data, sop_map_data_2):
testapp.post_json('/sop_map', sop_map_data, status=201)
res2chk = testapp.post_json('/sop_map', sop_map_data_2, status=201)
res_exp = testapp.post_json('/experiment_hi_c', experiment_data)
assert 'sop_mapping' in res_exp.json['@graph'][0]
assert res_exp.json['@graph'][0]['sop_mapping']['has_sop'] == "Yes"
assert res_exp.json['@graph'][0]['sop_mapping']['sop_map'] == res2chk.json['@graph'][0]['@id']
def test_experiment_update_hic_sop_mapping_to_v1_when_v2_deleted(
testapp, experiment_data, sop_map_data, sop_map_data_2):
res2chk = testapp.post_json('/sop_map', sop_map_data, status=201)
sop_map_data_2['status'] = 'deleted'
testapp.post_json('/sop_map', sop_map_data_2, status=201)
res_exp = testapp.post_json('/experiment_hi_c', experiment_data)
assert 'sop_mapping' in res_exp.json['@graph'][0]
assert res_exp.json['@graph'][0]['sop_mapping']['has_sop'] == "Yes"
assert res_exp.json['@graph'][0]['sop_mapping']['sop_map'] == res2chk.json['@graph'][0]['@id']
def test_experiment_update_hic_sop_map_not_added_when_already_present(testapp, experiment_data):
experiment_data['sop_mapping'] = {}
experiment_data['sop_mapping']['has_sop'] = 'No'
res = testapp.post_json('/experiment_hi_c', experiment_data)
assert 'sop_mapping' in res.json['@graph'][0]
assert res.json['@graph'][0]['sop_mapping']['has_sop'] == "No"
assert 'sop_map' not in res.json['@graph'][0]['sop_mapping']
def test_calculated_experiment_summary(testapp, experiment, mboI):
summary = 'in situ Hi-C on GM12878 with MboI'
res = testapp.patch_json(experiment['@id'], {'digestion_enzyme': mboI['@id']}, status=200)
assert res.json['@graph'][0]['experiment_summary'] == summary
assert summary in res.json['@graph'][0]['display_title']
def test_experiment_summary_repliseq(repliseq_4):
assert repliseq_4.get('experiment_summary') == '2-stage Repli-seq on GM12878 S-phase early'
# test for experiment_set_replicate _update function
def test_experiment_set_replicate_update_adds_experiments_in_set(testapp, experiment, replicate_experiment_set):
assert not replicate_experiment_set['experiments_in_set']
res = testapp.patch_json(
replicate_experiment_set['@id'],
{'replicate_exps':
[{'replicate_exp': experiment['@id'], 'bio_rep_no': 1, 'tec_rep_no': 1}]},
status=200)
assert experiment['@id'] in res.json['@graph'][0]['experiments_in_set']
# test for default_embedding practice with embedded list
# this test should change should any of the reference embeds below be altered
def test_experiment_set_default_embedded_list(registry, exp_types):
exp_data = {
'experiment_type': exp_types['microc']['uuid'],
'status': 'in review by lab'
}
# create experimentHiC obj; _update (and by extension, add_default_embeds)
# are called automatically
test_exp = ExperimentHiC.create(registry, None, exp_data)
# call reify embedded property (defined in snovault/resources.py)
embedded = test_exp.embedded
embedded_list = test_exp.embedded_list
type_info_embedded = registry[TYPES]['experiment_hi_c'].embedded_list
assert type_info_embedded == embedded_list
if 'produced_in_pub.*' in embedded_list:
assert 'produced_in_pub.*' in embedded
assert 'produced_in_pub.award.@id' in embedded
assert 'produced_in_pub.award.@type' in embedded
assert 'produced_in_pub.award.principals_allowed.*' in embedded
assert 'produced_in_pub.award.display_title' in embedded
assert 'produced_in_pub.award.uuid' in embedded
assert 'experiment_sets.accession' in embedded_list
assert 'experiment_sets.@id' in embedded
assert 'experiment_sets.@type' in embedded
assert 'experiment_sets.principals_allowed.*' in embedded
assert 'experiment_sets.display_title' in embedded
assert 'experiment_sets.uuid' in embedded
# tests for the experiment_sets calculated properties
def test_calculated_experiment_sets_for_custom_experiment_set(testapp, experiment, custom_experiment_set):
assert len(experiment['experiment_sets']) == 0
res = testapp.patch_json(custom_experiment_set['@id'], {'experiments_in_set': [experiment['@id']]}, status=200)
expt_res = testapp.get(experiment['@id'])
assert custom_experiment_set['uuid'] == expt_res.json['experiment_sets'][0]['uuid']
def test_calculated_experiment_sets_for_replicate_experiment_set(testapp, experiment, replicate_experiment_set):
assert len(experiment['experiment_sets']) == 0
res = testapp.patch_json(
replicate_experiment_set['@id'],
{'replicate_exps':
[{'replicate_exp': experiment['@id'], 'bio_rep_no': 1, 'tec_rep_no': 1}]},
status=200)
expt_res = testapp.get(experiment['@id'])
assert replicate_experiment_set['uuid'] == expt_res.json['experiment_sets'][0]['uuid']
@pytest.fixture
def pub1_data(lab, award):
# encode paper published 2012-09-06
return {
'award': award['@id'],
'lab': lab['@id'],
'ID': "PMID:22955616"
}
@pytest.fixture
def pub2_data(lab, award):
# Sanborn et al paper published 2015-11-24
return {
'award': award['@id'],
'lab': lab['@id'],
'ID': "PMID:26499245"
}
def test_calculated_produced_in_pub_for_rep_experiment_set(testapp, replicate_experiment_set, pub1_data):
# post single rep_exp_set to single pub
pub1_data['exp_sets_prod_in_pub'] = [replicate_experiment_set['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
expsetres = testapp.get(replicate_experiment_set['@id'])
assert 'produced_in_pub' in expsetres
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' in expsetres.json['produced_in_pub'].values()
def test_calculated_produced_in_pub_for_cust_experiment_set(testapp, custom_experiment_set, pub1_data):
# post single cust_exp_set to single pub
pub1_data['exp_sets_prod_in_pub'] = [custom_experiment_set['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
expsetres = testapp.get(custom_experiment_set['@id'])
assert 'produced_in_pub' in expsetres
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' in expsetres.json['produced_in_pub'].values()
def test_calculated_produced_in_pub_for_two_experiment_set_to_one_pub(
testapp, replicate_experiment_set, custom_experiment_set, pub1_data):
# post two exp_set to single pub
pub1_data['exp_sets_prod_in_pub'] = [replicate_experiment_set['@id'], custom_experiment_set['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
responses = [testapp.get(replicate_experiment_set['@id']),
testapp.get(custom_experiment_set['@id'])]
for response in responses:
assert 'produced_in_pub' in response
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' == response.json['produced_in_pub']['@id']
def test_calculated_produced_in_pub_for_two_experiment_set_two_pubs(
testapp, replicate_experiment_set, custom_experiment_set, pub1_data, pub2_data):
# post different exp_set to each pub
pub1_data['exp_sets_prod_in_pub'] = [replicate_experiment_set['@id']]
pub2_data['exp_sets_prod_in_pub'] = [custom_experiment_set['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
pub2res = testapp.post_json('/publication', pub2_data, status=201)
responses = [testapp.get(replicate_experiment_set['@id']),
testapp.get(custom_experiment_set['@id'])]
for response in responses:
assert 'produced_in_pub' in response
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' == responses[0].json['produced_in_pub']['@id']
assert '/publications/' + pub2res.json['@graph'][0]['uuid'] + '/' == responses[1].json['produced_in_pub']['@id']
def test_calculated_produced_in_pub_for_one_experiment_set_two_pubs(
testapp, replicate_experiment_set, pub1_data, pub2_data):
# post one exp_set to two pubs - this one should pick up only the most recent pub
pub1_data['exp_sets_prod_in_pub'] = [replicate_experiment_set['@id']]
pub2_data['exp_sets_prod_in_pub'] = [replicate_experiment_set['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
pub2res = testapp.post_json('/publication', pub2_data, status=201)
response = testapp.get(replicate_experiment_set['@id'])
assert 'produced_in_pub' in response
assert not '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' == response.json['produced_in_pub']['@id']
assert '/publications/' + pub2res.json['@graph'][0]['uuid'] + '/' == response.json['produced_in_pub']['@id']
def test_calculated_publications_in_experiment_set_no_data(
testapp, replicate_experiment_set, custom_experiment_set, pub1_data):
pub1res = testapp.post_json('/publication', pub1_data, status=201)
print(replicate_experiment_set)
print(custom_experiment_set)
assert not replicate_experiment_set['publications_of_set']
assert not custom_experiment_set['publications_of_set']
def test_calculated_publications_in_rep_experiment_set_2_fields(
testapp, replicate_experiment_set, pub1_data):
# post single rep_exp_set to single pub both fields
pub1_data['exp_sets_prod_in_pub'] = [replicate_experiment_set['@id']]
pub1_data['exp_sets_used_in_pub'] = [replicate_experiment_set['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
response = testapp.get(replicate_experiment_set['@id'])
print(response)
print('JSON:', response.json)
assert 'publications_of_set' in response
assert len(response.json['publications_of_set']) == 1
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' in response.json['publications_of_set'][0].values()
def test_calculated_publications_in_cust_experiment_set_used_in_field(
testapp, custom_experiment_set, pub1_data):
# post only used in publication one pub one exp set
pub1_data['exp_sets_used_in_pub'] = [custom_experiment_set['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
response = testapp.get(custom_experiment_set['@id'])
assert 'publications_of_set' in response
assert len(response.json['publications_of_set']) == 1
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' in response.json['publications_of_set'][0].values()
def test_calculated_publications_in_rep_experiment_set_two_pubs_both_fields(
testapp, replicate_experiment_set, pub1_data, pub2_data):
# post same experiment set to two pubs in either field
pub1_data['exp_sets_prod_in_pub'] = [replicate_experiment_set['@id']]
pub2_data['exp_sets_used_in_pub'] = [replicate_experiment_set['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
pub2res = testapp.post_json('/publication', pub2_data, status=201)
response = testapp.get(replicate_experiment_set['@id'])
assert 'publications_of_set' in response
assert len(response.json['publications_of_set']) == 2
publications = response.json['publications_of_set']
combined_pub_vals = [p['@id'] for p in publications]
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' in combined_pub_vals
assert '/publications/' + pub2res.json['@graph'][0]['uuid'] + '/' in combined_pub_vals
def test_calculated_publications_in_rep_experiment_set_two_pubs_in_used(
testapp, replicate_experiment_set, pub1_data, pub2_data):
# post same experiment set to two pubs in used in pub field
pub1_data['exp_sets_used_in_pub'] = [replicate_experiment_set['@id']]
pub2_data['exp_sets_used_in_pub'] = [replicate_experiment_set['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
pub2res = testapp.post_json('/publication', pub2_data, status=201)
response = testapp.get(replicate_experiment_set['@id'])
assert 'publications_of_set' in response
assert len(response.json['publications_of_set']) == 2
publications = response.json['publications_of_set']
combined_pub_vals = list(publications[0].values()) + list(publications[1].values())
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' in combined_pub_vals
assert '/publications/' + pub2res.json['@graph'][0]['uuid'] + '/' in combined_pub_vals
# experiment pub calculated properties tests
@pytest.fixture
def repset_w_exp1(testapp, replicate_experiment_set_data, experiment):
repset = replicate_experiment_set_data
repset['replicate_exps'] = [{'replicate_exp': experiment['@id'], 'bio_rep_no': 1, 'tec_rep_no': 1}]
return testapp.post_json('/experiment_set_replicate', repset).json['@graph'][0]
@pytest.fixture
def experiment2(testapp, experiment_data, exp_types):
experiment_data['experiment_type'] = exp_types['capc']['@id']
return testapp.post_json('/experiment_capture_c', experiment_data).json['@graph'][0]
@pytest.fixture
def custset_w_exp1(testapp, custom_experiment_set_data, experiment):
custset = custom_experiment_set_data
custset['experiments_in_set'] = [experiment['@id']]
return testapp.post_json('/experiment_set', custset).json['@graph'][0]
@pytest.fixture
def custset_w_exp2(testapp, custom_experiment_set_data, experiment2):
custset = custom_experiment_set_data
custset['experiments_in_set'] = [experiment2['@id']]
return testapp.post_json('/experiment_set', custset).json['@graph'][0]
def test_calculated_expt_produced_in_pub_for_rep_experiment_set(
testapp, repset_w_exp1, pub1_data):
# post single rep_exp_set to single pub
pub1_data['exp_sets_prod_in_pub'] = [repset_w_exp1['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
expres = testapp.get(repset_w_exp1['replicate_exps'][0]['replicate_exp'])
# import pdb; pdb.set_trace()
assert 'produced_in_pub' in expres
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' == expres.json['produced_in_pub']['@id']
def test_calculated_expt_produced_in_pub_for_expt_w_ref(
testapp, experiment_data, replicate_experiment_set_data, pub2_data, publication):
experiment_data['references'] = [publication['@id']]
# just check experiment by itself first
expt = testapp.post_json('/experiment_hi_c', experiment_data, status=201).json['@graph'][0]
assert 'produced_in_pub' in expt
assert publication['@id'] == expt['produced_in_pub']
# post repset with this experiment
replicate_experiment_set_data['replicate_exps'] = [{'bio_rep_no': 1, 'tec_rep_no': 1, 'replicate_exp': expt['@id']}]
repset = testapp.post_json('/experiment_set_replicate', replicate_experiment_set_data, status=201).json['@graph'][0]
# post single rep_exp_set to single pub
pub2_data['exp_sets_prod_in_pub'] = [repset['@id']]
testapp.post_json('/publication', pub2_data, status=201)
expinset = testapp.get(repset['replicate_exps'][0]['replicate_exp']).json
assert 'produced_in_pub' in expinset
assert publication['@id'] == expinset['produced_in_pub']['@id']
def test_calculated_expt_produced_in_pub_for_cust_experiment_set(
testapp, custset_w_exp1, pub1_data):
# post single cust_exp_set to single pub
pub1_data['exp_sets_prod_in_pub'] = [custset_w_exp1['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
expres = testapp.get(custset_w_exp1['experiments_in_set'][0])
assert 'produced_in_pub' not in expres.json.keys()
def test_calculated_expt_produced_in_pub_for_one_expt_in_two_expset_one_pub(
testapp, repset_w_exp1, custset_w_exp1, pub1_data):
# post two exp_set with same experiment (repset and custset) to single pub
pub1_data['exp_sets_prod_in_pub'] = [repset_w_exp1['@id'], custset_w_exp1['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
# both responses will get the same experiment
responses = [testapp.get(repset_w_exp1['replicate_exps'][0]['replicate_exp']),
testapp.get(custset_w_exp1['experiments_in_set'][0])]
for response in responses:
assert 'produced_in_pub' in response
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' == response.json['produced_in_pub']['@id']
def test_calculated_expt_produced_in_pub_for_two_exp_two_expset_two_pubs(
testapp, repset_w_exp1, custset_w_exp2, pub1_data, pub2_data):
# post 2 exp_set (one repset, one custom) each with diff expt to each pub
# only expt in repset should get the pub of repset
pub1_data['exp_sets_prod_in_pub'] = [repset_w_exp1['@id']]
pub2_data['exp_sets_prod_in_pub'] = [custset_w_exp2['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
testapp.post_json('/publication', pub2_data, status=201)
responses = [testapp.get(repset_w_exp1['replicate_exps'][0]['replicate_exp']),
testapp.get(custset_w_exp2['experiments_in_set'][0])]
assert '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' == responses[0].json['produced_in_pub']['@id']
assert 'produced_in_pub' not in responses[1].json
def test_calculated_expt_produced_in_pub_for_one_expt_one_expset_two_pubs(
testapp, repset_w_exp1, pub1_data, pub2_data):
# post one exp_set to two pubs - this one should pick up only the most recent pub
pub1_data['exp_sets_prod_in_pub'] = [repset_w_exp1['@id']]
pub2_data['exp_sets_prod_in_pub'] = [repset_w_exp1['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
pub2res = testapp.post_json('/publication', pub2_data, status=201)
response = testapp.get(repset_w_exp1['replicate_exps'][0]['replicate_exp'])
assert 'produced_in_pub' in response
assert not '/publications/' + pub1res.json['@graph'][0]['uuid'] + '/' == response.json['produced_in_pub']['@id']
assert '/publications/' + pub2res.json['@graph'][0]['uuid'] + '/' == response.json['produced_in_pub']['@id']
def test_calculated_publications_in_experiment_no_data(
testapp, repset_w_exp1, custset_w_exp2, pub1_data):
pub1res = testapp.post_json('/publication', pub1_data, status=201)
responses = [testapp.get(repset_w_exp1['replicate_exps'][0]['replicate_exp']),
testapp.get(custset_w_exp2['experiments_in_set'][0])]
for response in responses:
assert response.json['publications_of_exp'] == []
def test_calculated_publications_in_expt_w_repset_in_both_fields(
testapp, repset_w_exp1, pub1_data):
# post single rep_exp_set to single pub both fields
pub1_data['exp_sets_prod_in_pub'] = [repset_w_exp1['@id']]
pub1_data['exp_sets_used_in_pub'] = [repset_w_exp1['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
response = testapp.get(repset_w_exp1['replicate_exps'][0]['replicate_exp'])
assert 'publications_of_exp' in response
assert len(response.json['publications_of_exp']) == 1
assert pub1res.json['@graph'][0]['uuid'] == response.json['publications_of_exp'][0]['uuid']
def test_calculated_publications_in_expt_w_custset_used_in_field(
testapp, custset_w_exp2, pub1_data):
# post only used in publication one pub one exp set
pub1_data['exp_sets_used_in_pub'] = [custset_w_exp2['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
response = testapp.get(custset_w_exp2['experiments_in_set'][0])
assert 'publications_of_exp' in response
assert len(response.json['publications_of_exp']) == 1
assert pub1res.json['@graph'][0]['uuid'] == response.json['publications_of_exp'][0]['uuid']
def test_calculated_publications_in_expt_w_repset_two_pubs_both_fields(
testapp, repset_w_exp1, pub1_data, pub2_data):
# post same experiment set to two pubs in either field
pub1_data['exp_sets_prod_in_pub'] = [repset_w_exp1['@id']]
pub2_data['exp_sets_used_in_pub'] = [repset_w_exp1['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
pub2res = testapp.post_json('/publication', pub2_data, status=201)
pubuuids = [pub1res.json['@graph'][0]['uuid']]
pubuuids.append(pub2res.json['@graph'][0]['uuid'])
response = testapp.get(repset_w_exp1['replicate_exps'][0]['replicate_exp'])
assert 'publications_of_exp' in response
assert len(response.json['publications_of_exp']) == 2
publications = response.json['publications_of_exp']
for pub in publications:
assert pub['uuid'] in pubuuids
def test_calculated_publications_in_expt_w_repset_two_pubs_in_used(
testapp, repset_w_exp1, pub1_data, pub2_data):
# post same experiment set to two pubs in used in pub field
pub1_data['exp_sets_used_in_pub'] = [repset_w_exp1['@id']]
pub2_data['exp_sets_used_in_pub'] = [repset_w_exp1['@id']]
pub1res = testapp.post_json('/publication', pub1_data, status=201)
pub2res = testapp.post_json('/publication', pub2_data, status=201)
pubuuids = [pub1res.json['@graph'][0]['uuid']]
pubuuids.append(pub2res.json['@graph'][0]['uuid'])
response = testapp.get(repset_w_exp1['replicate_exps'][0]['replicate_exp'])
assert 'publications_of_exp' in response
assert len(response.json['publications_of_exp']) == 2
publications = response.json['publications_of_exp']
for pub in publications:
assert pub['uuid'] in pubuuids
def test_calculated_no_of_expts_in_set_w_no_exps(empty_replicate_set):
assert 'number_of_experiments' not in empty_replicate_set
def test_calculated_no_of_expts_in_set_w_2_exps(two_experiment_replicate_set):
assert two_experiment_replicate_set['number_of_experiments'] == 2
# tests for category calculated_property
@pytest.fixture
def target_w_prot(testapp, lab, award):
item = {
'description': "Protein target",
'targeted_proteins': ['CTCF (ABCD)'],
'award': award['@id'],
'lab': lab['@id'],
}
return testapp.post_json('/target', item).json['@graph'][0]
@pytest.fixture
def exp_w_target_info(lab, award, human_biosample, exp_types,
mboI, genomic_region_bio_feature):
return {
'lab': lab['@id'],
'award': award['@id'],
'biosample': human_biosample['@id'],
'experiment_type': exp_types['capc']['@id'],
'targeted_regions': [{'target': [genomic_region_bio_feature['@id']]}]
}
@pytest.fixture
def expt_w_targ_region(testapp, exp_w_target_info):
return testapp.post_json('/experiment_capture_c', exp_w_target_info).json['@graph'][0]
@pytest.fixture
def expt_w_2_targ_regions(testapp, exp_w_target_info, gene_bio_feature):
region = {'target': [gene_bio_feature['@id']]}
exp_w_target_info['targeted_regions'].append(region)
return testapp.post_json('/experiment_capture_c', exp_w_target_info).json['@graph'][0]
@pytest.fixture
def expt_w_target_data(lab, award, human_biosample,
prot_bio_feature, exp_types):
return {
'lab': lab['@id'],
'award': award['@id'],
'biosample': human_biosample['@id'],
'experiment_type': exp_types['chia']['@id'],
'targeted_factor': [prot_bio_feature['@id']]
}
@pytest.fixture
def expt_w_target(testapp, expt_w_target_data):
return testapp.post_json('/experiment_chiapet', expt_w_target_data).json['@graph'][0]
@pytest.fixture
def chipseq_expt(testapp, expt_w_target_data, exp_types):
expt_w_target_data['experiment_type'] = exp_types['chipseq']['@id']
return testapp.post_json('/experiment_seq', expt_w_target_data).json['@graph'][0]
@pytest.fixture
def tsaseq_expt(testapp, expt_w_target_data, exp_types):
expt_w_target_data['experiment_type'] = exp_types['tsaseq']['@id']
return testapp.post_json('/experiment_tsaseq', expt_w_target_data).json['@graph'][0]
@pytest.fixture
def repliseq_info(lab, award, human_biosample, exp_types):
return {
'lab': lab['@id'],
'award': award['@id'],
'biosample': human_biosample['@id'],
'experiment_type': exp_types['repliseq']['@id'],
}
@pytest.fixture
def repliseq_1(testapp, repliseq_info):
return testapp.post_json('/experiment_repliseq', repliseq_info).json['@graph'][0]
@pytest.fixture
def repliseq_2(testapp, repliseq_info):
repliseq_info['stage_fraction'] = 'early'
return testapp.post_json('/experiment_repliseq', repliseq_info).json['@graph'][0]
@pytest.fixture
def repliseq_3(testapp, repliseq_info):
repliseq_info['stage_fraction'] = 'early'
repliseq_info['total_fractions_in_exp'] = 16
return testapp.post_json('/experiment_repliseq', repliseq_info).json['@graph'][0]
@pytest.fixture
def repliseq_4(testapp, repliseq_info):
repliseq_info['stage_fraction'] = 'early'
repliseq_info['total_fractions_in_exp'] = 2
repliseq_info['cell_cycle_phase'] = 'S'
return testapp.post_json('/experiment_repliseq', repliseq_info).json['@graph'][0]
@pytest.fixture
def experiment_atacseq(testapp, repliseq_info, exp_types):
repliseq_info['experiment_type'] = exp_types['atacseq']['@id']
return testapp.post_json('/experiment_atacseq', repliseq_info).json['@graph'][0]
@pytest.fixture
def damid_no_fusion(testapp, repliseq_info, exp_types):
repliseq_info['experiment_type'] = exp_types['dam']['@id']
return testapp.post_json('/experiment_damid', repliseq_info).json['@graph'][0]
@pytest.fixture
def damid_w_fusion(testapp, repliseq_info, prot_bio_feature, exp_types):
repliseq_info['experiment_type'] = exp_types['dam']['@id']
repliseq_info['targeted_factor'] = [prot_bio_feature['@id']]
return testapp.post_json('/experiment_damid', repliseq_info).json['@graph'][0]
@pytest.fixture
def damid_w_multifusion(testapp, repliseq_info, prot_bio_feature, gene_bio_feature, exp_types):
repliseq_info['experiment_type'] = exp_types['dam']['@id']
repliseq_info['targeted_factor'] = [prot_bio_feature['@id'], gene_bio_feature['@id']]
return testapp.post_json('/experiment_damid', repliseq_info).json['@graph'][0]
@pytest.fixture
def basic_info(lab, award):
return {
'lab': lab['@id'],
'award': award['@id'],
}
@pytest.fixture
def imaging_path_1(testapp, basic_info, genomic_region_bio_feature):
basic_info['target'] = [genomic_region_bio_feature['@id']]
basic_info['labeled_probe'] = 'FITC goat anti rabbit'
return testapp.post_json('/imaging_path', basic_info).json['@graph'][0]
@pytest.fixture
def imaging_path_2(testapp, basic_info, genomic_region_bio_feature):
basic_info['target'] = [genomic_region_bio_feature['@id']]
basic_info['labeled_probe'] = 'TRITC horse anti rabbit'
return testapp.post_json('/imaging_path', basic_info).json['@graph'][0]
@pytest.fixture
def imaging_path_3(testapp, basic_info, basic_region_bio_feature):
basic_info['target'] = [basic_region_bio_feature['@id']]
basic_info['labeled_probe'] = 'DAPI'
return testapp.post_json('/imaging_path', basic_info).json['@graph'][0]
@pytest.fixture
def microscopy_no_path(testapp, repliseq_info, exp_types):
repliseq_info['experiment_type'] = exp_types['fish']['@id']
return testapp.post_json('/experiment_mic', repliseq_info).json['@graph'][0]
@pytest.fixture
def microscopy_w_path(testapp, repliseq_info, imaging_path_1, exp_types):
repliseq_info['experiment_type'] = exp_types['fish']['@id']
img_path = {'path': imaging_path_1['@id'], 'channel': 'ch01'}
repliseq_info['imaging_paths'] = [img_path]
return testapp.post_json('/experiment_mic', repliseq_info).json['@graph'][0]
@pytest.fixture
def microscopy_w_multipath(testapp, repliseq_info, imaging_path_1, imaging_path_2,
imaging_path_3, exp_types):
repliseq_info['experiment_type'] = exp_types['fish']['@id']
img_path1 = {'path': imaging_path_1['@id'], 'channel': 'ch01'}
img_path2 = {'path': imaging_path_2['@id'], 'channel': 'ch02'}
img_path3 = {'path': imaging_path_3['@id'], 'channel': 'ch03'}
repliseq_info['imaging_paths'] = [img_path1, img_path2, img_path3]
return testapp.post_json('/experiment_mic', repliseq_info).json['@graph'][0]
@pytest.fixture
def microscopy_w_splitpath(testapp, repliseq_info, exp_types,
imaging_path_1, imaging_path_3,
basic_region_bio_feature, genomic_region_bio_feature):
'''Sometimes a (group of) target(s) is split into different imaging paths,
e.g. due to multiplexing. If text is formatted as follows, the split group
will be found and replaced with the sum'''
repliseq_info['experiment_type'] = exp_types['fish']['@id']
img_path1 = {'path': imaging_path_1['@id'], 'channel': 'ch01'}
img_path3 = {'path': imaging_path_3['@id'], 'channel': 'ch03'}
repliseq_info['imaging_paths'] = [img_path1, img_path3]
testapp.patch_json(basic_region_bio_feature['@id'],
{'preferred_label': '15 TADs on chr19'}).json['@graph'][0]
testapp.patch_json(genomic_region_bio_feature['@id'],
{'preferred_label': '22 TADs on chr19'}).json['@graph'][0]
return testapp.post_json('/experiment_mic', repliseq_info).json['@graph'][0]
def test_experiment_atacseq_display_title(experiment_atacseq):
assert experiment_atacseq.get('display_title') == 'ATAC-seq on GM12878 - ' + experiment_atacseq.get('accession')
def test_experiment_damid_w_multifusion_display_title(damid_w_multifusion):
assert damid_w_multifusion.get('display_title') == 'DamID-seq with mulitiple DAM fusions on GM12878 - ' + damid_w_multifusion.get('accession')
def test_experiment_chiapet_w_target_display_title(expt_w_target):
assert expt_w_target.get('display_title') == 'ChIA-PET against RAD21 protein on GM12878 - ' + expt_w_target.get('accession')
def test_experiment_chipseq_w_target_display_title(chipseq_expt):
assert chipseq_expt.get('display_title') == 'ChIP-seq against RAD21 protein on GM12878 - ' + chipseq_expt.get('accession')
def test_experiment_tsaseq_display_title(tsaseq_expt):
assert tsaseq_expt.get('display_title') == 'TSA-seq against RAD21 protein on GM12878 - ' + tsaseq_expt.get('accession')
def test_experiment_categorizer_4_mic_no_path(testapp, microscopy_no_path):
assert microscopy_no_path['experiment_categorizer']['field'] == 'Default'
assert microscopy_no_path['experiment_categorizer'].get('value') is None
def test_experiment_categorizer_4_mic_w_path(testapp, microscopy_w_path, genomic_region_bio_feature):
assert microscopy_w_path['experiment_categorizer']['field'] == 'Target'
assert microscopy_w_path['experiment_categorizer']['value'] == genomic_region_bio_feature['display_title']
def test_experiment_categorizer_4_mic_w_multi_path(testapp, microscopy_w_multipath, genomic_region_bio_feature, basic_region_bio_feature):
vals2chk = [genomic_region_bio_feature['display_title'], basic_region_bio_feature['display_title']]
len2chk = len(vals2chk[0]) + len(vals2chk[1]) + 2
assert microscopy_w_multipath['experiment_categorizer']['field'] == 'Target'
value = microscopy_w_multipath['experiment_categorizer']['value']
assert len(value) == len2chk
for v in vals2chk:
assert v in value
def test_experiment_categorizer_4_mic_w_split_path(testapp, microscopy_w_splitpath):
'''Sometimes a (group of) target(s) is split into different imaging paths,
e.g. due to multiplexing. Sum the split targets and return only one string.'''
assert microscopy_w_splitpath['experiment_categorizer']['value'] == '37 TADs on chr19'
def test_experiment_categorizer_4_chiapet_no_fusion(testapp, repliseq_info, exp_types):
repliseq_info['experiment_type'] = exp_types['chia']['@id']
res = testapp.post_json('/experiment_chiapet', repliseq_info).json['@graph'][0]
assert res['experiment_categorizer']['field'] == 'Default'
assert res['experiment_categorizer']['value'] is None
def test_experiment_categorizer_4_damid_no_fusion(testapp, damid_no_fusion):
assert damid_no_fusion['experiment_categorizer']['field'] == 'Target'
assert damid_no_fusion['experiment_categorizer'].get('value') == 'None (Control)'
def test_experiment_categorizer_4_damid_w_fusion(testapp, damid_w_fusion, prot_bio_feature):
assert damid_w_fusion['experiment_categorizer']['field'] == 'Target'
assert damid_w_fusion['experiment_categorizer']['value'] == prot_bio_feature['display_title']
def test_experiment_categorizer_4_repliseq_no_fraction_info(testapp, repliseq_1):
assert repliseq_1['experiment_categorizer']['field'] == 'Default'
assert repliseq_1['experiment_categorizer'].get('value') is None
def test_experiment_categorizer_4_repliseq_only_fraction(testapp, repliseq_2):
wanted = 'early of an unspecified number of fractions'
assert repliseq_2['experiment_categorizer']['field'] == 'Fraction'
assert repliseq_2['experiment_categorizer']['value'] == wanted
def test_experiment_categorizer_4_repliseq_fraction_and_total(testapp, repliseq_3):
wanted = 'early of 16 fractions'
assert repliseq_3['experiment_categorizer']['field'] == 'Fraction'
assert repliseq_3['experiment_categorizer']['value'] == wanted
def test_experiment_categorizer_w_target(testapp, expt_w_target, prot_bio_feature):
assert expt_w_target['experiment_categorizer']['field'] == 'Target'
assert expt_w_target['experiment_categorizer']['value'] == prot_bio_feature['display_title']
def test_experiment_categorizer_w_enzyme(testapp, experiment, mboI):
assert experiment['experiment_categorizer']['field'] == 'Enzyme'
assert experiment['experiment_categorizer']['value'] == mboI['display_title']
def test_experiment_categorizer_w_target_and_enzyme(testapp, expt_w_target, prot_bio_feature, mboI):
# import pdb; pdb.set_trace()
res = testapp.patch_json(expt_w_target['@id'], {'digestion_enzyme': mboI['@id']}).json['@graph'][0]
assert res['digestion_enzyme'] == mboI['@id']
assert res['experiment_categorizer']['field'] == 'Target'
assert res['experiment_categorizer']['value'] == prot_bio_feature['display_title']
def test_experiment_categorizer_w_no_cat1(testapp, experiment_data, exp_types):
del experiment_data['digestion_enzyme']
experiment_data['experiment_type'] = exp_types['rnaseq']['@id']
expt = testapp.post_json('/experiment_seq', experiment_data).json['@graph'][0]
assert expt['experiment_categorizer']['field'] == 'Default'
assert expt['experiment_categorizer'].get('value') is None
def test_experiment_categorizer_cap_c_no_regions(testapp, experiment_data, mboI, exp_types):
experiment_data['experiment_type'] = exp_types['capc']['@id']
expt = testapp.post_json('/experiment_capture_c', experiment_data).json['@graph'][0]
assert expt['experiment_categorizer']['field'] == 'Enzyme'
assert expt['experiment_categorizer']['value'] == mboI['display_title']
def test_experiment_categorizer_cap_c_w_region(expt_w_targ_region, genomic_region_bio_feature):
assert expt_w_targ_region['experiment_categorizer']['field'] == 'Target'
assert expt_w_targ_region['experiment_categorizer']['value'] == genomic_region_bio_feature['display_title']
def test_experiment_categorizer_cap_c_w_2regions(
expt_w_2_targ_regions, genomic_region_bio_feature, gene_bio_feature):
wanted = ', '.join(sorted([genomic_region_bio_feature['display_title'], gene_bio_feature['display_title']]))
assert expt_w_2_targ_regions['experiment_categorizer']['field'] == 'Target'
assert expt_w_2_targ_regions['experiment_categorizer']['value'] == wanted
@pytest.fixture
def new_exp_type(lab, award):
data = {
'uuid': str(uuid4()),
'title': 'Title',
'lab': lab['@id'],
'award': award['@id'],
'status': 'released',
'valid_item_types': ['ExperimentSeq']
}
return data
def test_validate_exp_type_valid(testapp, experiment_data, new_exp_type):
exp_type1 = testapp.post_json('/experiment_type', new_exp_type).json['@graph'][0]
experiment_data['experiment_type'] = exp_type1['@id']
expt = testapp.post_json('/experiment_hi_c', experiment_data, status=422)
testapp.patch_json(exp_type1['@id'], {'valid_item_types': ['ExperimentSeq', 'ExperimentHiC']})
expt = testapp.post_json('/experiment_hi_c', experiment_data, status=201).json['@graph'][0]
assert expt['experiment_type'] == '/experiment-types/title/'
def test_validate_experiment_set_duplicate_replicate_experiments(testapp, rep_set_data, experiment):
rep_set_data['replicate_exps'] = [{'bio_rep_no': 1, 'tec_rep_no': 1, 'replicate_exp': experiment['@id']},
{'bio_rep_no': 1, 'tec_rep_no': 2, 'replicate_exp': experiment['@id']}]
repset = testapp.post_json('/experiment_set_replicate', rep_set_data, status=422)
assert repset.json['errors'][0]['name'] == 'ExperimentSet: non-unique exps'
assert 'Duplicate experiment' in repset.json['errors'][0]['description']
|
|
'''tzinfo timezone information for Australia/NSW.'''
from pytz.tzinfo import DstTzInfo
from pytz.tzinfo import memorized_datetime as d
from pytz.tzinfo import memorized_ttinfo as i
class NSW(DstTzInfo):
'''Australia/NSW timezone definition. See datetime.tzinfo for details'''
zone = 'Australia/NSW'
_utc_transition_times = [
d(1,1,1,0,0,0),
d(1916,12,31,14,1,0),
d(1917,3,24,15,0,0),
d(1941,12,31,16,0,0),
d(1942,3,28,15,0,0),
d(1942,9,26,16,0,0),
d(1943,3,27,15,0,0),
d(1943,10,2,16,0,0),
d(1944,3,25,15,0,0),
d(1971,10,30,16,0,0),
d(1972,2,26,16,0,0),
d(1972,10,28,16,0,0),
d(1973,3,3,16,0,0),
d(1973,10,27,16,0,0),
d(1974,3,2,16,0,0),
d(1974,10,26,16,0,0),
d(1975,3,1,16,0,0),
d(1975,10,25,16,0,0),
d(1976,3,6,16,0,0),
d(1976,10,30,16,0,0),
d(1977,3,5,16,0,0),
d(1977,10,29,16,0,0),
d(1978,3,4,16,0,0),
d(1978,10,28,16,0,0),
d(1979,3,3,16,0,0),
d(1979,10,27,16,0,0),
d(1980,3,1,16,0,0),
d(1980,10,25,16,0,0),
d(1981,2,28,16,0,0),
d(1981,10,24,16,0,0),
d(1982,4,3,16,0,0),
d(1982,10,30,16,0,0),
d(1983,3,5,16,0,0),
d(1983,10,29,16,0,0),
d(1984,3,3,16,0,0),
d(1984,10,27,16,0,0),
d(1985,3,2,16,0,0),
d(1985,10,26,16,0,0),
d(1986,3,15,16,0,0),
d(1986,10,18,16,0,0),
d(1987,3,14,16,0,0),
d(1987,10,24,16,0,0),
d(1988,3,19,16,0,0),
d(1988,10,29,16,0,0),
d(1989,3,18,16,0,0),
d(1989,10,28,16,0,0),
d(1990,3,3,16,0,0),
d(1990,10,27,16,0,0),
d(1991,3,2,16,0,0),
d(1991,10,26,16,0,0),
d(1992,2,29,16,0,0),
d(1992,10,24,16,0,0),
d(1993,3,6,16,0,0),
d(1993,10,30,16,0,0),
d(1994,3,5,16,0,0),
d(1994,10,29,16,0,0),
d(1995,3,4,16,0,0),
d(1995,10,28,16,0,0),
d(1996,3,30,16,0,0),
d(1996,10,26,16,0,0),
d(1997,3,29,16,0,0),
d(1997,10,25,16,0,0),
d(1998,3,28,16,0,0),
d(1998,10,24,16,0,0),
d(1999,3,27,16,0,0),
d(1999,10,30,16,0,0),
d(2000,3,25,16,0,0),
d(2000,8,26,16,0,0),
d(2001,3,24,16,0,0),
d(2001,10,27,16,0,0),
d(2002,3,30,16,0,0),
d(2002,10,26,16,0,0),
d(2003,3,29,16,0,0),
d(2003,10,25,16,0,0),
d(2004,3,27,16,0,0),
d(2004,10,30,16,0,0),
d(2005,3,26,16,0,0),
d(2005,10,29,16,0,0),
d(2006,4,1,16,0,0),
d(2006,10,28,16,0,0),
d(2007,3,24,16,0,0),
d(2007,10,27,16,0,0),
d(2008,3,29,16,0,0),
d(2008,10,25,16,0,0),
d(2009,3,28,16,0,0),
d(2009,10,24,16,0,0),
d(2010,3,27,16,0,0),
d(2010,10,30,16,0,0),
d(2011,3,26,16,0,0),
d(2011,10,29,16,0,0),
d(2012,3,24,16,0,0),
d(2012,10,27,16,0,0),
d(2013,3,30,16,0,0),
d(2013,10,26,16,0,0),
d(2014,3,29,16,0,0),
d(2014,10,25,16,0,0),
d(2015,3,28,16,0,0),
d(2015,10,24,16,0,0),
d(2016,3,26,16,0,0),
d(2016,10,29,16,0,0),
d(2017,3,25,16,0,0),
d(2017,10,28,16,0,0),
d(2018,3,24,16,0,0),
d(2018,10,27,16,0,0),
d(2019,3,30,16,0,0),
d(2019,10,26,16,0,0),
d(2020,3,28,16,0,0),
d(2020,10,24,16,0,0),
d(2021,3,27,16,0,0),
d(2021,10,30,16,0,0),
d(2022,3,26,16,0,0),
d(2022,10,29,16,0,0),
d(2023,3,25,16,0,0),
d(2023,10,28,16,0,0),
d(2024,3,30,16,0,0),
d(2024,10,26,16,0,0),
d(2025,3,29,16,0,0),
d(2025,10,25,16,0,0),
d(2026,3,28,16,0,0),
d(2026,10,24,16,0,0),
d(2027,3,27,16,0,0),
d(2027,10,30,16,0,0),
d(2028,3,25,16,0,0),
d(2028,10,28,16,0,0),
d(2029,3,24,16,0,0),
d(2029,10,27,16,0,0),
d(2030,3,30,16,0,0),
d(2030,10,26,16,0,0),
d(2031,3,29,16,0,0),
d(2031,10,25,16,0,0),
d(2032,3,27,16,0,0),
d(2032,10,30,16,0,0),
d(2033,3,26,16,0,0),
d(2033,10,29,16,0,0),
d(2034,3,25,16,0,0),
d(2034,10,28,16,0,0),
d(2035,3,24,16,0,0),
d(2035,10,27,16,0,0),
d(2036,3,29,16,0,0),
d(2036,10,25,16,0,0),
d(2037,3,28,16,0,0),
d(2037,10,24,16,0,0),
]
_transition_info = [
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
i(36000,0,'EST'),
i(39600,3600,'EST'),
]
NSW = NSW()
|
End of preview. Expand
in Dataset Viewer.
- Downloads last month
- 63