Skip to content

Commit 130a127

Browse files
fschlimbGitHub Enterprise
fschlimb
authored and
GitHub Enterprise
committed
make it work on win, MPI having issues still
1 parent 8f3ce30 commit 130a127

14 files changed

+286
-187
lines changed

‎conda-recipe/bld.bat

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
if "%PY3K%"=="1" (
2+
set ARGS=""
3+
) else {
4+
set ARGS="--old-and-unmanageable"
5+
}
6+
7+
set DAAL4PY_VERSION=%PKG_VERSION%
8+
set CNCROOT=%PREFIX%
9+
set TBBROOT=%PREFIX%
10+
set DAALROOT=%PREFIX%
11+
12+
%PYTHON% setup.py install %ARGS%

‎conda-recipe/build.sh

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
#!/bin/bash
22

3-
if [ -n "$PY_VER" ] && [ "${PY_VER:0:1}" -lt "3" ]; then
4-
ARG="--old-and-unmanageable"
3+
if [ "$PY3K" == "1" ]; then
4+
ARGS=""
55
else
6-
ARG=""
6+
ARGS="--old-and-unmanageable"
77
fi
8-
DAAL4PY_VERSION=$PKG_VERSION CNCROOT=${PREFIX} TBBROOT=${PREFIX} DAALROOT=${PREFIX} ${PYTHON} setup.py install $ARG
8+
9+
DAAL4PY_VERSION=$PKG_VERSION CNCROOT=${PREFIX} TBBROOT=${PREFIX} DAALROOT=${PREFIX} ${PYTHON} setup.py install $ARGS

‎conda-recipe/meta.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ requirements:
1717
- tbb-devel
1818
- cnc >=1.2.300
1919
- daal-devel >=2018.0.1
20+
- daal-static >=2018.0.1 [win]
2021
- cython
2122
- jinja2
2223
run:
@@ -27,7 +28,8 @@ requirements:
2728
- daal >=2018.0.1
2829

2930
build:
30-
number: {{buildnumber}}
31+
number: {{buildnumber}}
32+
include_recipe: False
3133

3234
test:
3335
imports:

‎examples/run_examples.py

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#*******************************************************************************
2+
# Copyright 2014-2017 Intel Corporation
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#******************************************************************************/
16+
17+
import os
18+
import platform
19+
import struct
20+
import subprocess
21+
import sys
22+
23+
from os.path import join as jp
24+
from time import gmtime, strftime
25+
26+
exdir = os.path.dirname(os.path.realpath(__file__))
27+
28+
IS_WIN = platform.system() == 'Windows'
29+
30+
assert 8 * struct.calcsize('P') in [32, 64]
31+
32+
if 8 * struct.calcsize('P') == 32:
33+
logdir = jp(exdir, '_results', 'ia32')
34+
else:
35+
logdir = jp(exdir, '_results', 'intel64')
36+
37+
def get_exe_cmd(ex):
38+
if 'batch' in ex:
39+
return '"' + sys.executable + '" "' + ex + '"'
40+
else:
41+
return 'mpirun -n 4 -genv DIST_CNC=MPI "' + sys.executable + '" "' + ex + '"'
42+
43+
def run_all():
44+
success = 0
45+
n = 0
46+
if not os.path.exists(logdir):
47+
os.makedirs(logdir)
48+
for (dirpath, dirnames, filenames) in os.walk(exdir):
49+
for script in filenames:
50+
if script.endswith('.py') and script not in ['run_examples.py', '__init__.py']:
51+
n += 1
52+
logfn = jp(logdir, script.replace('.py', '.res'))
53+
with open(logfn, 'w') as logfile:
54+
print('\n##### ' + jp(dirpath, script))
55+
execute_string = get_exe_cmd(jp(dirpath, script))
56+
proc = subprocess.Popen(execute_string if IS_WIN else ['/bin/bash', '-c', execute_string],
57+
stdout=subprocess.PIPE,
58+
stderr=subprocess.STDOUT,
59+
shell=(True if IS_WIN else False))
60+
out = proc.communicate()[0]
61+
logfile.write(out.decode('ascii'))
62+
if proc.returncode:
63+
print(out)
64+
print(strftime("%H:%M:%S", gmtime()) + '\tFAILED\t' + script + '\twith errno\t' + str(proc.returncode))
65+
else:
66+
success += 1
67+
print(strftime("%H:%M:%S", gmtime()) + '\tPASSED\t' + script)
68+
if success != n:
69+
print('{}/{} examples passed, {} failed'.format(success,n, n - success))
70+
print('Error(s) occured. Logs can be found in ' + logdir)
71+
else:
72+
print('{}/{} examples passed'.format(success,n))
73+
74+
if __name__ == '__main__':
75+
run_all()

‎generator/gen_daal4py.py

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,7 @@ def read(self):
150150
what namespace it is affiliated with. Once done, we have a dictionary where the key is the namespace
151151
and the values are namespace class objects. These objects carry all information as extracted by parse.py.
152152
"""
153+
print('reading headers from ' + self.include_root)
153154
for (dirpath, dirnames, filenames) in os.walk(self.include_root):
154155
for filename in filenames:
155156
if filename.endswith('.h') and not 'neural_networks' in dirpath and not any(filename.endswith(x) for x in cython_interface.ignore_files):
@@ -288,7 +289,7 @@ def to_hltype(self, ns, t):
288289
if t.endswith('ModelPtr'):
289290
thens = self.get_ns(ns, t, attrs=['typedefs'])
290291
return ('daal::' + thens + '::ModelPtr', 'class', tns)
291-
if t in ['data_management::NumericTablePtr',] or t in ifaces.values():
292+
if t in ['data_management::NumericTablePtr'] or t in ifaces.values():
292293
return ('daal::' + t, 'class', tns)
293294
if 'Batch' in self.namespace_dict[ns].classes and t in self.namespace_dict[ns].classes['Batch'].typedefs:
294295
tns, tname = splitns(self.namespace_dict[ns].classes['Batch'].typedefs[t])
@@ -325,7 +326,7 @@ def get_values(self, ns, n):
325326
if nns:
326327
nn = splitns(n)[1]
327328
if nn in self.namespace_dict[nns].enums:
328-
return [nns + '::' + x for x in self.namespace_dict[nns].enums[nn]]
329+
return [re.sub(r'(?<!daal::)algorithms::', r'daal::algorithms::', nns + '::' + x) for x in self.namespace_dict[nns].enums[nn]]
329330
return ['unknown_' + nns + '_class_'+n]
330331
return ['unknown_'+n]
331332

@@ -341,7 +342,7 @@ def get_tmplarg(self, ns, n):
341342
if nns:
342343
nn = splitns(n)[1]
343344
if nn in self.namespace_dict[nns].enums:
344-
return nns + '::' + nn
345+
return re.sub(r'(?<!daal::)algorithms::', r'daal::algorithms::', nns + '::' + nn)
345346
return 'unknown_' + nns + '_class_'+n
346347
return 'unknown_'+n
347348

@@ -460,7 +461,8 @@ def prepare_modelmaps(self, ns, mname='Model'):
460461
if not any(g.endswith(x) for x in ['SerializationTag',]):
461462
gn = splitns(g)[1].replace('get', '')
462463
if not any(gn == x[1] for x in jparams['named_gets']):
463-
jparams['named_gets'].append((huhu[g], gn))
464+
typ = re.sub(r'(?<!daal::)data_management', r'daal::data_management', huhu[g])
465+
jparams['named_gets'].append((typ, gn))
464466
return jparams
465467

466468

@@ -555,7 +557,8 @@ def prepare_hlwrapper(self, ns, mode, func):
555557
'template_args': None,
556558
'pargs': None})
557559
for s in v['specs']:
558-
tdecl.append({'template_decl': OrderedDict([(x, v['tmpl_decl'][x]) for x in s['template_decl']]),
560+
tdecl.append({'template_decl': OrderedDict([(re.sub(r'(?<!daal::)algorithms::', r'daal::algorithms::', x),
561+
v['tmpl_decl'][x]) for x in s['template_decl']]),
559562
'template_args': [s['expl'][x] if x in s['expl'] else x for x in v['tmpl_decl']],
560563
'pargs': [s['expl'][x] for x in s['expl']]})
561564
else:
@@ -624,6 +627,7 @@ def prepare_hlwrapper(self, ns, mode, func):
624627
pval = tmp
625628
if pval != None:
626629
thetype = (hlt if hlt else parms[p])
630+
pval = re.sub(r'(?<!daal::)algorithms::', r'daal::algorithms::', pval)
627631
if tmp in jparams['params_req']:
628632
td['params_req'][tmp] = pval
629633
decl_req.append('const ' + thetype + ' ' + tmp)
@@ -645,7 +649,6 @@ def prepare_hlwrapper(self, ns, mode, func):
645649
tmp_iargs_decl = []
646650
tmp_iargs_call = []
647651
tmp_input_args = []
648-
setinputs = ''
649652
inp = self.get_class_for_typedef(ns, 'Batch', 'InputType')
650653
if not inp and 'Input' in self.namespace_dict[ns].classes:
651654
inp = (ns, 'Input')
@@ -665,6 +668,7 @@ def prepare_hlwrapper(self, ns, mode, func):
665668
if 'NumericTablePtr' in itype:
666669
#ns in has_dist and iname in has_dist[ns]['step_specs'][0].inputnames or iname in ['data', 'labels', 'dependentVariable', 'tableToFill']:
667670
itype = 'TableOrFList *'
671+
ins = re.sub(r'(?<!daal::)algorithms::', r'daal::algorithms::', ins)
668672
tmp_iargs_decl.insert(i, 'const ' + itype + ' ' + iname + dflt)
669673
tmp_iargs_call.insert(i, iname)
670674
tmp_input_args.insert(i, (ins + '::' + iname, iname, itype))
@@ -752,7 +756,7 @@ def hlapi(self, algo_patterns):
752756
for e in self.namespace_dict[ns].enums:
753757
for v in self.namespace_dict[ns].enums[e]:
754758
vv = ns + '::' + v
755-
cpp_begin += ' '*4 +'{"' + v + '", ' + vv + '},\n'
759+
cpp_begin += ' '*4 +'{"' + v + '", daal::' + vv + '},\n'
756760
cpp_begin += '};\n\n'
757761

758762
hlargs = {}
@@ -798,7 +802,6 @@ def gen_daal4py(daalroot, outdir, warn_all=False):
798802
global no_warn
799803
if warn_all:
800804
no_warn = {}
801-
802805
iface = cython_interface(jp(daalroot, 'include', 'algorithms'))
803806
iface.read()
804807
cpp_h, cpp_cpp, pyx_file = iface.hlapi(['kmeans',

‎generator/wrapper_gen.py

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ def my_procid():
151151
{% for m in enum_gets %}
152152
extern "C" {{m[2]}} {{'*' if 'Ptr' in m[2] else ''}} get_{{flatname}}_{{m[1]}}({{class_type}} * obj_)
153153
{
154-
return RAW< {{m[2]}} >()((*obj_)->get({{m[0]}}::{{m[1]}}));
154+
return RAW< {{m[2]}} >()((*obj_)->get(daal::{{m[0]}}::{{m[1]}}));
155155
}
156156
{% endfor %}
157157
{% for m in named_gets %}
@@ -280,17 +280,17 @@ def __dealloc__(self):
280280
{% macro gen_typedefs(ns, template_decl, template_args, mode="Batch", suffix="b", step_spec=None) %}
281281
{% set disttarg = (step_spec.name.rsplit('__', 1)[0] + ', ') if step_spec.name else "" %}
282282
{% if template_decl|length > 0 %}
283-
typedef {{ns}}::{{mode}}<{{disttarg + ', '.join(template_args)}}> algo{{suffix}}_type;
283+
typedef daal::{{ns}}::{{mode}}<{{disttarg + ', '.join(template_args)}}> algo{{suffix}}_type;
284284
{% else %}
285-
typedef {{ns}}::{{mode}} algo{{suffix}}_type;
285+
typedef daal::{{ns}}::{{mode}} algo{{suffix}}_type;
286286
{% endif %}
287287
{% if step_spec %}
288288
typedef {{step_spec.iomanager}}< algo{{suffix}}_type, {{', '.join(step_spec.input)}}, {{step_spec.output}}{{(","+",".join(step_spec.iomargs)) if step_spec.iomargs else ""}} > iom{{suffix}}_type;
289289
{% else %}
290290
{% if iombatch %}
291291
typedef {{iombatch}} iom{{suffix}}_type;
292292
{% else %}
293-
typedef IOManager< algo{{suffix}}_type, services::SharedPtr< typename algo{{suffix}}_type::InputType >, services::SharedPtr< typename algo{{suffix}}_type::ResultType > > iom{{suffix}}_type;
293+
typedef IOManager< algo{{suffix}}_type, daal::services::SharedPtr< typename algo{{suffix}}_type::InputType >, daal::services::SharedPtr< typename algo{{suffix}}_type::ResultType > > iom{{suffix}}_type;
294294
{% endif %}
295295
{% endif %}
296296
{%- endmacro %}
@@ -397,7 +397,7 @@ def __dealloc__(self):
397397
{{algo}}__iface__(bool d=false) : _distributed(d) {}
398398
{% set indent = 23+(result_map.class_type|length) %}
399399
virtual {{result_map.class_type}} * compute({{(',\n'+' '*indent).join(iargs_decl|cppdecl)}},
400-
{{' '*indent}}bool setup_only = false) {assert(false);}
400+
{{' '*indent}}bool setup_only = false) {assert(false); return NULL;}
401401
};
402402
"""
403403

@@ -622,11 +622,12 @@ def setup(self,
622622
{% endfor %}
623623
{%- endmacro %}
624624
625-
extern "C" {{algo}}__iface__ * mk_{{algo}}({{pargs_decl|cpp_decl(pargs_call, template_decl, 56+2*(algo|length))}})
625+
extern "C" {{algo}}__iface__ * mk_{{algo}}({{pargs_decl|cpp_decl(pargs_call, template_decl, 37+2*(algo|length))}})
626626
{
627627
{% if template_decl %}
628628
{{tfactory(template_decl.items()|list, algo+'_manager', pargs_call, dist=dist)}}
629-
throw std::invalid_argument("no equivalent(s) for C++ template argument(s) in mk_{{algo}}");
629+
std::cerr << "no equivalent(s) for C++ template argument(s) exist in mk_{{algo}}" << std::endl;
630+
return NULL;
630631
{% else %}
631632
return new {{algo}}_manager({{', '.join(pargs_call)}}, distributed);
632633
{% endif %}
@@ -781,9 +782,9 @@ def __flat(typ):
781782
else:
782783
r = '_'.join(nn)
783784
return ('c_' if cpp and typ.endswith('__iface__') else '') + r + (' *' if cpp and any(typ.endswith(x) for x in ['__iface__', 'Ptr']) else '')
784-
ty = ty.replace('daal::algorithms::kernel_function::KernelIfacePtr', 'services::SharedPtr<kernel_function::KernelIface>')
785-
ty = re.sub(r'(daal::)?(algorithms::)?(engines::)?EnginePtr', r'services::SharedPtr<engines::BatchBase>', ty)
786-
ty = re.sub(r'(?:daal::)?(?:algorithms::)?([^:]+::)BatchPtr', r'services::SharedPtr<\1Batch>', ty)
785+
ty = ty.replace('daal::algorithms::kernel_function::KernelIfacePtr', 'daal::services::SharedPtr<kernel_function::KernelIface>')
786+
ty = re.sub(r'(daal::)?(algorithms::)?(engines::)?EnginePtr', r'daal::services::SharedPtr<engines::BatchBase>', ty)
787+
ty = re.sub(r'(?:daal::)?(?:algorithms::)?([^:]+::)BatchPtr', r'daal::services::SharedPtr<\1Batch>', ty)
787788
ty = re.sub(r'(daal::)?services::SharedPtr<([^>]+)>', r'\2__iface__', ty)
788789
return ' '.join([__flat(x).replace('const', '') for x in ty.split(' ')])
789790
return [_flat(x) for x in t] if isinstance(t,list) else _flat(t)

0 commit comments

Comments
 (0)