In [1]:
%load_ext noworkflow
nip = %now_ip
def first(x):
return next(x)
In [2]:
%now_sql_schema
Out[2]:
create table trial (
id INTEGER PRIMARY KEY AUTOINCREMENT,
start TIMESTAMP,
finish TIMESTAMP,
script TEXT,
code_hash TEXT,
arguments TEXT,
inherited_id INTEGER, -- Id of the prospective tuple that we are inheriting module information (due to --bypass-modules)
parent_id INTEGER, -- Id of the parent trial that is used to create the history
run INTEGER, -- trial created through now run command
FOREIGN KEY (inherited_id) REFERENCES trial ON DELETE RESTRICT,
FOREIGN KEY (parent_id) REFERENCES trial ON DELETE SET NULL
);
CREATE INDEX trial_inherited_id on trial(inherited_id);
CREATE INDEX trial_parent_id on trial(parent_id);
create table head (
id INTEGER PRIMARY KEY AUTOINCREMENT,
script TEXT,
trial_id INTEGER,
FOREIGN KEY (trial_id) REFERENCES trial ON DELETE SET NULL
);
CREATE INDEX head_trial_id on head(trial_id);
create table module (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
version TEXT,
path TEXT,
code_hash TEXT
);
create table dependency (
trial_id INTEGER NOT NULL,
module_id INTEGER NOT NULL,
PRIMARY KEY (trial_id, module_id),
FOREIGN KEY (trial_id) REFERENCES trial ON DELETE CASCADE,
FOREIGN KEY (module_id) REFERENCES module ON DELETE CASCADE
);
CREATE INDEX dependency_trial_id on dependency(trial_id);
CREATE INDEX dependency_module_id on dependency(module_id);
create table function_def (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
code_hash TEXT,
trial_id INTEGER,
FOREIGN KEY (trial_id) REFERENCES trial ON DELETE CASCADE
);
CREATE INDEX function_def_trial_id on function_def(trial_id);
create table object (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
type TEXT CHECK (type IN ('GLOBAL', 'ARGUMENT', 'FUNCTION_CALL')),
function_def_id INTEGER,
FOREIGN KEY (function_def_id) REFERENCES function_def ON DELETE CASCADE
);
CREATE INDEX object_function_def_id on object(function_def_id);
create table environment_attr (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
value TEXT,
trial_id INTEGER,
FOREIGN KEY (trial_id) REFERENCES trial ON DELETE CASCADE
);
CREATE INDEX environment_attr_trial_id on environment_attr(trial_id);
create table function_activation (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
line INTEGER,
return TEXT,
start TIMESTAMP,
finish TIMESTAMP,
caller_id INTEGER,
trial_id INTEGER,
FOREIGN KEY (caller_id) REFERENCES function_activation ON DELETE CASCADE,
FOREIGN KEY (trial_id) REFERENCES trial ON DELETE CASCADE
);
CREATE INDEX function_activation_caller_id on function_activation(caller_id);
CREATE INDEX function_activation_trial_id on function_activation(trial_id);
create table object_value (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
value TEXT,
type TEXT CHECK (type IN ('GLOBAL', 'ARGUMENT')),
function_activation_id INTEGER,
FOREIGN KEY (function_activation_id) REFERENCES function_activation ON DELETE CASCADE
);
CREATE INDEX object_value_function_activation_id on object_value(function_activation_id);
create table file_access (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
mode TEXT,
buffering TEXT,
content_hash_before TEXT,
content_hash_after TEXT,
timestamp TIMESTAMP,
function_activation_id INTEGER,
trial_id INTEGER,
FOREIGN KEY (function_activation_id) REFERENCES function_activation ON DELETE CASCADE,
FOREIGN KEY (trial_id) REFERENCES trial ON DELETE CASCADE
);
CREATE INDEX file_access_function_activation_id on file_access(function_activation_id);
CREATE INDEX file_access_trial_id on file_access(trial_id);
-- Slicing
create table slicing_variable (
trial_id INTEGER,
vid INTEGER,
name TEXT,
line INTEGER,
value TEXT,
time TIMESTAMP,
FOREIGN KEY (trial_id) REFERENCES trial ON DELETE CASCADE,
PRIMARY KEY (trial_id, vid)
);
CREATE INDEX slicing_variable_trial_id on slicing_variable(trial_id);
CREATE INDEX slicing_variable_vid on slicing_variable(vid);
create table slicing_usage (
trial_id INTEGER,
id INTEGER,
vid INTEGER,
name TEXT,
line INTEGER,
FOREIGN KEY (trial_id, vid) REFERENCES slicing_variable(trial_id, vid) ON DELETE CASCADE,
PRIMARY KEY (trial_id, id)
);
CREATE INDEX slicing_usage_trial_id on slicing_usage(trial_id);
CREATE INDEX slicing_usage_vid on slicing_usage(vid);
create table slicing_dependency (
trial_id INTEGER,
id INTEGER,
dependent INTEGER,
supplier INTEGER,
FOREIGN KEY (trial_id, dependent) REFERENCES slicing_variable(trial_id, vid) ON DELETE CASCADE,
FOREIGN KEY (trial_id, supplier) REFERENCES slicing_variable(trial_id, vid) ON DELETE CASCADE,
PRIMARY KEY (trial_id, id)
);
CREATE INDEX slicing_dependency_trial_id on slicing_dependency(trial_id);
CREATE INDEX slicing_dependency_dependent on slicing_dependency(dependent);
CREATE INDEX slicing_dependency_supplier on slicing_dependency(supplier);
-- Cache
create table graph_cache (
id INTEGER PRIMARY KEY AUTOINCREMENT,
type TEXT,
name TEXT,
attributes TEXT,
content_hash TEXT,
duration INTEGER,
timestamp TIMESTAMP
);
In [3]:
%now_prolog_schema
Out[3]:
%
% FACT: activation(trial_id, id, name, start, finish, caller_activation_id).
%
%
% FACT: access(trial_id, id, name, mode, content_hash_before, content_hash_after, timestamp, activation_id).
%
%
% FACT: variable(trial_id, vid, name, line, value, timestamp).
%
%
% FACT: usage(trial_id, id, vid, name, line).
%
%
% FACT: dependency(trial_id, id, dependent, supplier).
%
%
% ID-BASED ACCESSOR RULES FOR
% activation(trial_id, id, name, start, finish, caller_activation_id).
% access(trial_id, id, name, mode, content_hash_before, content_hash_after, timestamp, activation_id).
%
name(_, [], []).
name(Trial_id, [Id|Ids], [Name|Names]) :- name(Trial_id, Id, Name), name(Trial_id, Ids, Names).
name(Trial_id, Id, Name) :- activation(Trial_id, Id, Name, _, _, _).
name(Trial_id, Id, Name) :- access(Trial_id, Id, Name, _, _, _, _, _).
timestamp_id(Trial_id, Id, Start, start) :- activation(Trial_id, Id, _, Start, _, _).
timestamp_id(Trial_id, Id, Finish, finish) :- activation(Trial_id, Id, _, _, Finish, _).
timestamp_id(Trial_id, Id, Timestamp) :- access(Trial_id, Id, _, _, _, _, Timestamp, _).
duration_id(Trial_id, Id, Duration) :- timestamp_id(Trial_id, Id, Start, start), timestamp_id(Trial_id, Id, Finish, finish), Duration is Finish - Start.
successor_id(Trial_id, Before, After) :- timestamp_id(Trial_id, Before, TS1, start), timestamp_id(Trial_id, After, TS2, finish), TS1 =< TS2.
successor_id(Trial_id, Before, After) :- timestamp_id(Trial_id, Before, TS1), timestamp_id(Trial_id, After, TS2), TS1 =< TS2.
activation_id(Trial_id, Caller, Called) :- activation(Trial_id, Called, _, _, _, Caller).
mode_id(Trial_id, Id, Mode) :- access(Trial_id, Id, _, Mode, _, _, _, _).
file_read_id(Trial_id, Id) :- mode_id(Trial_id, Id, Mode), atom_prefix(Mode, 'r').
file_written_id(Trial_id, Id) :- mode_id(Trial_id, Id, Mode), atom_prefix(Mode, 'w').
hash_id(Trial_id, Id, Hash, before) :- access(Trial_id, Id, _, _, Hash, _, _, _).
hash_id(Trial_id, Id, Hash, after) :- access(Trial_id, Id, _, _, _, Hash, _, _).
changed_id(Trial_id, Id) :- hash_id(Trial_id, Id, Hash1, before), hash_id(Trial_id, Id, Hash2, after), Hash1 \== Hash2.
access_id(Trial_id, Function, File) :- access(Trial_id, File, _, _, _, _, _, Function).
%
% ID-BASED INFERENCE RULES
%
activation_stack_id(Trial_id, Called, []) :- activation_id(Trial_id, nil, Called).
activation_stack_id(Trial_id, Called, [Caller|Callers]) :- activation_id(Trial_id, Caller, Called), activation_stack_id(Trial_id, Caller, Callers).
indirect_activation_id(Trial_id, Caller, Called) :- activation_stack_id(Trial_id, Called, Callers), member(Caller, Callers).
% Naive! Should check arguments and return values (program slicing?) to avoid false positives
activation_influence_id(Trial_id, Influencer, Influenced) :- successor_id(Trial_id, Influencer, Influenced).
access_stack_id(Trial_id, File, [Function|Functions]) :- access_id(Trial_id, Function, File), activation_stack_id(Trial_id, Function, Functions).
indirect_access_id(Trial_id, Function, File) :- access_stack_id(Trial_id, File, Functions), member(Function, Functions).
access_influence_id(Trial_id, Influencer, Influenced) :- file_read_id(Trial_id, Influencer), file_written_id(Trial_id, Influenced), successor_id(Trial_id, Influencer, Influenced), access_id(Trial_id, F1, Influencer), access_id(Trial_id, F2, Influenced), activation_influence_id(Trial_id, F1, F2).
%
% NAME-BASED ACCESSOR RULES
%
timestamp(Trial_id, Name, Timestamp, Moment) :- timestamp_id(Trial_id, Id, Timestamp, Moment), name(Trial_id, Id, Name).
timestamp(Trial_id, Name, Timestamp) :- timestamp_id(Trial_id, Id, Timestamp), name(Trial_id, Id, Name).
duration(Trial_id, Name, Duration) :- duration_id(Trial_id, Id, Duration), name(Trial_id, Id, Name).
successor(Trial_id, Before, After) :- successor_id(Trial_id, Before_id, After_id), name(Trial_id, Before_id, Before), name(Trial_id, After_id, After).
mode(Trial_id, Name, Mode) :- mode_id(Trial_id, Id, Mode), name(Trial_id, Id, Name).
file_read(Trial_id, Name) :- file_read_id(Trial_id, Id), name(Trial_id, Id, Name).
file_written(Trial_id, Name) :- file_written_id(Trial_id, Id), name(Trial_id, Id, Name).
hash(Trial_id, Name, Hash, Moment) :- hash_id(Trial_id, Id, Hash, Moment), name(Trial_id, Id, Name).
changed(Trial_id, Name) :- changed_id(Trial_id, Id), name(Trial_id, Id, Name).
%
% NAME-BASED INFERENCE RULES
%
activation_stack(Trial_id, Called, Callers) :- activation_stack_id(Trial_id, Called_id, Caller_ids), name(Trial_id, Called_id, Called), name(Trial_id, Caller_ids, Callers).
indirect_activation(Trial_id, Caller, Called) :- indirect_activation_id(Trial_id, Caller_id, Called_id), name(Trial_id, Called_id, Called), name(Trial_id, Caller_id, Caller).
activation_influence(Trial_id, Influencer, Influenced) :- activation_influence_id(Trial_id, Influencer_id, Influenced_id), name(Trial_id, Influencer_id, Influencer), name(Trial_id, Influenced_id, Influenced).
access_stack(Trial_id, File, Functions) :- access_stack_id(Trial_id, File_id, Functions_id), name(Trial_id, File_id, File), name(Trial_id, Functions_id, Functions).
indirect_access(Trial_id, Function, File) :- indirect_access_id(Trial_id, Function_id, File_id), name(Trial_id, Function_id, Function), name(Trial_id, File_id, File).
access_influence(Trial_id, Influencer, Influenced) :- access_influence_id(Trial_id, Influencer_id, Influenced_id), name(Trial_id, Influencer_id, Influencer), name(Trial_id, Influenced_id, Influenced).
%
% SLICING-BASED ACCESSOR RULES
%
dep(Trial_id, Dependent, Supplier) :- dependency(Trial_id, _, Dependent, Supplier).
usage_or_assign(Trial_id, Name, Line, Id) :- usage(Trial_id, _, Id, Name, Line).
usage_or_assign(Trial_id, Name, Line, Id) :- variable(Trial_id, Id, Name, Line, _, _).
var_name(Trial_id, Id, Name) :- variable(Trial_id, Id, Name, _, _, _).
var_line(Trial_id, Id, Line) :- variable(Trial_id, Id, _, Line, _, _).
var_info(Trial_id, Id, variable(Trial_id, Id, Name, Line, Value)) :- variable(Trial_id, Id, Name, Line, Value, _).
%
% SLICING-BASED INFERENCE RULES
%
slice(_, [],[]).
slice(Trial_id, [Id|L1], L2) :- slice(Trial_id, Id, L3), slice(Trial_id, L1, L4), union(L3, L4, L2), !.
slice(Trial_id, Id, [Id|L1]) :- bagof(X, dep(Trial_id, Id, X),L2), !, slice(Trial_id, L2, L1).
slice(_, Id, [Id]).
slice_name(Trial_id, Id, Names) :- slice(Trial_id, Id, X), maplist(var_name(Trial_id), X, Names).
slice_line(Trial_id, Id, Lines) :- slice(Trial_id, Id, X), maplist(var_line(Trial_id), X, Lines).
slice_info(Trial_id, Id, Infos) :- slice(Trial_id, Id, X), maplist(var_info(Trial_id), X, Infos).
In [4]:
def extract(trial_id, filename, month):
t = nip.Trial(trial_id)
sql = first(nip.persistence.query("""
SELECT name, content_hash_before
FROM file_access
WHERE trial_id = {}
AND name = "{}"
""".format(trial_id, filename)))
fhash = sql['content_hash_before']
content = nip.persistence.get(fhash)
with open('.temp.dat', 'w') as f:
f.write(content)
result = !./precipitation.py .temp.dat $month
return sum(map(float,result[0].split(';')))
In [5]:
extract(18, 'p13.dat', 2)
Out[5]:
78.1
In [ ]:
%
Content source: gems-uff/noworkflow
Similar notebooks: