forked from hasktorch/hasktorch
-
Notifications
You must be signed in to change notification settings - Fork 0
/
get-deps.sh
executable file
·222 lines (191 loc) · 7.62 KB
/
get-deps.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
#!/usr/bin/env bash
# - gets submodules recursively to get pytorch and inline-c fork repo dependencies
# - Retrieves a prebuilt libtorch binary per https://pytorch.org/cppdocs/installing.html
# - Retrieves a release binary for mkl https://github.com/intel/mkl-dnn/releases
# which is a runtime dependency that is not package w/ libtorch
set -eu
USE_NIGHTLY=0
USE_BINARY_FOR_CI=0
COMPUTE_ARCH=cpu
SKIP_DOWNLOAD=0
VERSION=1.11.0
if ! command -v unzip &> /dev/null
then
echo "unzip could not be found. Please install unzip ("sudo apt install unzip" for ubuntu systems)"
exit
fi
if ! command -v wget &> /dev/null
then
echo "wget could not be found. Please install wget ("sudo apt install wget" for ubuntu systems)"
exit
fi
if ! command -v git &> /dev/null
then
echo "git could not be found. Please install git ("sudo apt install git-all" for ubuntu systems)"
exit
fi
if ! command -v sed &> /dev/null
then
echo "sed could not be found. Please install git ("sudo apt install sed" for ubuntu systems)"
exit
fi
if ! command -v python &> /dev/null
then
echo "python could not be found. Please install python ("sudo apt install python3.6" for ubuntu systems)"
exit
fi
if ! command -v pip &> /dev/null
then
echo "pip could not be found. Please install python ("sudo apt install python-pip" for ubuntu systems)"
exit
fi
usage_exit() {
echo "Usage: $0 [-n] [-c] [-a "cpu" or "cu102" or "cu113"] [-s]" 1>&2
echo " -n # Use nightly libtorch w/ -l" 1>&2
echo " # Use libtorch-$(VERSION) w/o -l" 1>&2
echo "" 1>&2
echo " -c # Download libtorch from hasktorch's site w/ -c" 1>&2
echo " # Download libtorch from pytorch's site w/o -c" 1>&2
echo "" 1>&2
echo " -a cpu # Use CPU without CUDA" 1>&2
echo " -a cu102 # Use CUDA-10.2" 1>&2
echo " -a cu113 # Use CUDA-11.3" 1>&2
echo "" 1>&2
echo " -s # Skip download" 1>&2
echo "" 1>&2
echo " -h # Show this help" 1>&2
exit 1
}
while getopts nca:sh OPT
do
case $OPT in
n) USE_NIGHTLY=1
;;
c) USE_BINARY_FOR_CI=1
;;
a) COMPUTE_ARCH=$OPTARG
;;
s) SKIP_DOWNLOAD=1
;;
h) usage_exit
;;
\?) usage_exit
;;
esac
done
if [ "$SKIP_DOWNLOAD" = 0 ] ; then
# git submodule update --init --recursive
# pytorch v1.3 has unlinked submodule of https://github.com/IvanKobzarev/fbjni.git
# For now, we can not update recursively.
git submodule update --init
case "$(uname)" in
"Darwin")
if [ "$USE_NIGHTLY" = 1 ] ; then
wget https://download.pytorch.org/libtorch/nightly/cpu/libtorch-macos-latest.zip
unzip libtorch-macos-latest.zip
rm libtorch-macos-latest.zip
elif [ "$USE_BINARY_FOR_CI" = 1 ] ; then
wget https://github.com/hasktorch/libtorch-binary-for-ci/releases/download/${VERSION}/cpu-libtorch-macos-latest.zip
unzip cpu-libtorch-macos-latest.zip
rm cpu-libtorch-macos-latest.zip
else
wget https://download.pytorch.org/libtorch/cpu/libtorch-macos-${VERSION}.zip
unzip libtorch-macos-${VERSION}.zip
rm libtorch-macos-${VERSION}.zip
fi
wget https://github.com/intel/mkl-dnn/releases/download/v0.17.2/mklml_mac_2019.0.1.20181227.tgz
tar -xzf mklml_mac_2019.0.1.20181227.tgz
rm -f mklml_mac_2019.0.1.20181227.tgz
rm -f mklml_mac_2019.0.1.20181227.tgz.1
rm -rf mklml
mv mklml_mac_2019.0.1.20181227 mklml
wget https://github.com/hasktorch/tokenizers/releases/download/libtokenizers-v0.1/libtokenizers-macos.zip
unzip libtokenizers-macos.zip
;;
"Linux")
if [ "$USE_NIGHTLY" = 1 ] ; then
wget https://download.pytorch.org/libtorch/nightly/${COMPUTE_ARCH}/libtorch-cxx11-abi-shared-with-deps-latest.zip
unzip libtorch-cxx11-abi-shared-with-deps-latest.zip
rm libtorch-cxx11-abi-shared-with-deps-latest.zip
elif [ "$USE_BINARY_FOR_CI" = 1 ] ; then
wget https://github.com/hasktorch/libtorch-binary-for-ci/releases/download/${VERSION}/${COMPUTE_ARCH}-libtorch-cxx11-abi-shared-with-deps-latest.zip
unzip ${COMPUTE_ARCH}-libtorch-cxx11-abi-shared-with-deps-latest.zip
rm ${COMPUTE_ARCH}-libtorch-cxx11-abi-shared-with-deps-latest.zip
else
case "${COMPUTE_ARCH}" in
"cpu" ) URL=https://download.pytorch.org/libtorch/${COMPUTE_ARCH}/libtorch-cxx11-abi-shared-with-deps-${VERSION}%2Bcpu.zip ;;
"cu102" ) URL=https://download.pytorch.org/libtorch/${COMPUTE_ARCH}/libtorch-cxx11-abi-shared-with-deps-${VERSION}%2Bcu102.zip ;;
"cu113" ) URL=https://download.pytorch.org/libtorch/${COMPUTE_ARCH}/libtorch-cxx11-abi-shared-with-deps-${VERSION}%2Bcu113.zip ;;
*)
1>&2 printf "Error: invalid value '%s' passed to -a\n\n" "$COMPUTE_ARCH"
usage_exit
esac
wget -O libtorch-cxx11-abi-shared-with-deps-${VERSION}.zip "$URL"
unzip libtorch-cxx11-abi-shared-with-deps-${VERSION}.zip
rm libtorch-cxx11-abi-shared-with-deps-${VERSION}.zip
fi
wget https://github.com/intel/mkl-dnn/releases/download/v0.17.2/mklml_lnx_2019.0.1.20181227.tgz
tar -xzf mklml_lnx_2019.0.1.20181227.tgz
rm -f mklml_lnx_2019.0.1.20181227.tgz
rm -f mklml_lnx_2019.0.1.20181227.tgz.1
rm -rf mklml
mv mklml_lnx_2019.0.1.20181227 mklml
ln -s libmklml_intel.so mklml/lib/libmklml.so
wget https://github.com/hasktorch/tokenizers/releases/download/libtokenizers-v0.1/libtokenizers-linux.zip
unzip libtokenizers-linux.zip
;;
esac
fi
# Following codes are copied from pytorch/tools/run-clang-tidy-in-ci.sh.
# Generate ATen files.
echo "Generate ATen files."
if [ ! -e pytorch ] ; then
git clone https://github.com/pytorch/pytorch.git
else
pushd pytorch
git pull origin v$VERSION
popd
fi
pushd pytorch
git checkout v$VERSION
if [[ ! -d build ]]; then
mkdir build
fi
PYTHON=python
if ! (python --version | grep "Python 2") ;then
PYTHON=python3
fi
if ! ($PYTHON -c 'import yaml') ; then
$PYTHON -m pip install --user pyyaml
fi
if ! ($PYTHON -c 'import dataclasses') ; then
$PYTHON -m pip install --user dataclasses
fi
if ! ($PYTHON -c 'import typing_extensions') ; then
$PYTHON -m pip install --user typing_extensions
fi
# See https://github.com/pytorch/pytorch/blob/master/.circleci/scripts/cpp_doc_push_script.sh
$PYTHON -m tools.codegen.gen \
-s aten/src/ATen \
-d build/aten/src/ATen
# Sanitize "name: n" fields to be strings rather than booleans in Declarations.yaml
case "$(uname)" in
"Darwin")
sed -i '' -e "s/::std::/std::/g" build/aten/src/ATen/Declarations.yaml
sed -i '' -e "s/ name: n$/ name: 'n'/g" -e "s/ name: N$/ name: 'N'/g" build/aten/src/ATen/Declarations.yaml
sed -i '' -e "s/ name: t$/ name: 't'/g" -e "s/ name: T$/ name: 'T'/g" build/aten/src/ATen/Declarations.yaml
sed -i '' -e "s/ name: y$/ name: 'y'/g" -e "s/ name: Y$/ name: 'Y'/g" build/aten/src/ATen/Declarations.yaml
sed -i '' -e "s/ default: \([^'].*\)$/ default: '\1'/g" build/aten/src/ATen/Declarations.yaml
;;
"Linux")
sed -i -e "s/::std::/std::/g" build/aten/src/ATen/Declarations.yaml
sed -i -e "s/ name: n$/ name: 'n'/g" -e "s/ name: N$/ name: 'N'/g" build/aten/src/ATen/Declarations.yaml
sed -i -e "s/ name: t$/ name: 't'/g" -e "s/ name: T$/ name: 'T'/g" build/aten/src/ATen/Declarations.yaml
sed -i -e "s/ name: y$/ name: 'y'/g" -e "s/ name: Y$/ name: 'Y'/g" build/aten/src/ATen/Declarations.yaml
sed -i -e "s/ default: \([^'].*\)$/ default: '\1'/g" build/aten/src/ATen/Declarations.yaml
;;
esac
popd
pushd ../spec
ln -fs ../deps/pytorch/build/aten/src/ATen/Declarations.yaml
popd