From 70d47f92db5eecb50c9e860e0ea0e80490f4f74e Mon Sep 17 00:00:00 2001 From: Runtian Zhou Date: Thu, 2 Aug 2018 08:44:00 -0700 Subject: [PATCH 01/16] Add support for rand_like op in fusion compiler (#9795) Summary: Enabled support for generating random numbers in fusion compiler. Currently a philox RNG implemented by Tensorflow is used, as the NVRTC couldn't resolve the curand.h header correctly. The two implementation should have exact same behavior according to our tests. Pull Request resolved: https://github.com/pytorch/pytorch/pull/9795 Differential Revision: D8999029 Pulled By: SsnL fbshipit-source-id: f0d2616a699a942e2f370bdb02ac77b9c463d7b8 --- test/test_jit.py | 23 ++++ torch/csrc/jit/fusion_compiler.cpp | 174 ++++++++++++++++++++++++-- torch/csrc/jit/fusion_compiler.h | 4 +- torch/csrc/jit/passes/graph_fuser.cpp | 1 + 4 files changed, 190 insertions(+), 12 deletions(-) diff --git a/test/test_jit.py b/test/test_jit.py index 3a78dcbab4133..a2d18184947f9 100644 --- a/test/test_jit.py +++ b/test/test_jit.py @@ -438,6 +438,29 @@ def f(x, y): ge = self.checkTrace(f, (x, y)) self.assertExpectedGraph(ge.graph_for(x, y)) + @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") + @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + def test_fusion_rand(self): + class M(torch.jit.ScriptModule): + __constants__ = ['d'] + + def __init__(self): + self.d = torch.device('cuda') + + @torch.jit.script_method + def create(self, x): + return x * x + x + torch.rand_like(x) + + x = torch.zeros([3, 4, 5], dtype=torch.float, device='cuda') + m = M() + out1 = m.create(x) + out2 = m.create(x) + self.assertNotEqual(out1, out2) + self.assertTrue(torch.all(out1 >= 0)) + self.assertTrue(torch.all(out1 < 1)) + self.assertTrue(torch.all(out2 >= 0)) + self.assertTrue(torch.all(out2 < 1)) + @staticmethod def fn_test_comparison_gt_lt(x, y): mask = (x > 0).type_as(x) diff --git a/torch/csrc/jit/fusion_compiler.cpp b/torch/csrc/jit/fusion_compiler.cpp index 22f8b40ba3054..dad50440944e8 100644 --- a/torch/csrc/jit/fusion_compiler.cpp +++ b/torch/csrc/jit/fusion_compiler.cpp @@ -15,6 +15,7 @@ #ifdef USE_CUDA #include "ATen/cuda/CUDAContext.h" #include "THC/THC.h" +#include #include "torch/csrc/cuda/cuda_check.h" #include #include @@ -30,6 +31,10 @@ #include #include +#ifdef USE_CUDA +THCGenerator* THCRandom_getGenerator(THCState* state); +#endif + namespace torch { namespace jit { std::vector TensorDesc::findContiguous( @@ -78,6 +83,7 @@ typedef signed char int8_t; typedef short int int16_t; typedef long long int int64_t; ${HalfHeader} +${RandHeader} #endif typedef ${IndexType} IndexType; template @@ -88,11 +94,113 @@ struct TensorInfo { }; )"); +// We rewrite the code for philox RNG from curand as nvrtc couldn't resolve the +// curand header correctly. +constexpr auto rand_support_literal = R"( + + class Philox { + public: + __device__ inline Philox(unsigned long long seed, + unsigned long long subsequence, + unsigned long long offset) { + key.x = (unsigned int)seed; + key.y = (unsigned int)(seed >> 32); + counter = make_uint4(0, 0, 0, 0); + counter.z = (unsigned int)(subsequence); + counter.w = (unsigned int)(subsequence >> 32); + STATE = 0; + incr_n(offset / 4); + } + + __device__ inline unsigned long operator()() { + if(STATE == 0) { + uint4 counter_ = counter; + uint2 key_ = key; + for(int i = 0; i < 9; i++) { + counter_ = single_round(counter_, key_); + key_.x += (kPhilox10A); key_.y += (kPhilox10B); + } + output = single_round(counter_, key_); + incr(); + } + unsigned long ret; + switch(STATE) { + case 0: ret = output.x; break; + case 1: ret = output.y; break; + case 2: ret = output.z; break; + case 3: ret = output.w; break; + } + STATE = (STATE + 1) % 4; + return ret; + } + + private: + uint4 counter; + uint4 output; + uint2 key; + unsigned int STATE; + __device__ inline void incr_n(unsigned long long n) { + unsigned int nlo = (unsigned int)(n); + unsigned int nhi = (unsigned int)(n >> 32); + counter.x += nlo; + if (counter.x < nlo) + nhi++; + counter.y += nhi; + if (nhi <= counter.y) + return; + if (++counter.z) + return; + ++counter.w; + } + __device__ inline void incr() { + if (++counter.x) + return; + if (++counter.y) + return; + if (++counter.z) + return; + ++counter.w; + } + __device__ unsigned int mulhilo32(unsigned int a, unsigned int b, + unsigned int *result_high) { + *result_high = __umulhi(a, b); + return a*b; + } + + __device__ inline uint4 single_round(uint4 ctr, uint2 key) { + unsigned int hi0; + unsigned int hi1; + unsigned int lo0 = mulhilo32(kPhiloxSA, ctr.x, &hi0); + unsigned int lo1 = mulhilo32(kPhiloxSB, ctr.z, &hi1); + + uint4 ret = {hi1 ^ ctr.y ^ key.x, lo1, hi0 ^ ctr.w ^ key.y, lo0}; + return ret; + } + + static const unsigned long kPhilox10A = 0x9E3779B9; + static const unsigned long kPhilox10B = 0xBB67AE85; + static const unsigned long kPhiloxSA = 0xD2511F53; + static const unsigned long kPhiloxSB = 0xCD9E8D57; + }; + + // Inverse of 2^32. + #define M_RAN_INVM32 2.3283064e-10f + __device__ __inline__ float uniform(unsigned int x) { + return x * M_RAN_INVM32; + } +)"; + +constexpr auto rand_param = ",unsigned long long seed, unsigned long long offset"; +constexpr auto rand_init = R"( + int idx = blockIdx.x*blockDim.x + threadIdx.x; + Philox rnd(seed, idx, offset); +)"; auto cuda_compilation_unit_template = CodeTemplate(R"( ${type_declarations} extern "C" __global__ -void ${kernelName}(IndexType totalElements, ${formals}) { +void ${kernelName}(IndexType totalElements, ${formals} ${RandParam}) { + ${RandInit} for (IndexType linearIndex = blockIdx.x * blockDim.x + threadIdx.x; linearIndex < totalElements; linearIndex += gridDim.x * blockDim.x) { @@ -280,9 +388,10 @@ std::string encodeRHS(Node * n) { {aten::remainder, "remainderf(${0}, ${1})"}, {aten::pow, "powf(${0}, ${1})"}, - // binary with alpha + //alpha {aten::add, "${0} + ${2}*${1}"}, {aten::sub, "(${0} - ${2}*${1})"}, + {aten::rand_like, "uniform(rnd())"}, // simple derivatives {aten::_sigmoid_backward, "${0} * ${1} * (1.f - ${1})"}, @@ -309,10 +418,12 @@ std::string encodeRHS(Node * n) { return format(str, env); } -std::vector emitCompilationUnit(std::ostream & out, - const std::string & name, - AnnotatedGraph & agraph, - bool use_cuda) { +std::pair, bool> emitCompilationUnit( + std::ostream& out, + const std::string& name, + AnnotatedGraph& agraph, + bool use_cuda) { + bool has_random = false; Graph& subgraph = *agraph.graph; TemplateEnv env; env.s("kernelName",name); @@ -385,6 +496,11 @@ std::vector emitCompilationUnit(std::ostream & out, // FusedConcat nodes work by narrowing the output Tensors before the kernel runs if (n->kind() == prim::FusedConcat) continue; + if(n->kind() == aten::rand_like) { + has_random = true; + if(!use_cuda) + throw std::runtime_error("Fusion doesn't support rand on CPU"); + } env.s("node",valueName(n->output())); env.s("rhs", encodeRHS(n)); body << format("auto ${node} = ${rhs};\n",env); @@ -412,6 +528,16 @@ std::vector emitCompilationUnit(std::ostream & out, env.s("HalfHeader", ""); } + if (has_random) { + env.s("RandHeader", rand_support_literal); + env.s("RandParam", rand_param); + env.s("RandInit", rand_init); + } else { + env.s("RandHeader", ""); + env.s("RandParam", ""); + env.s("RandInit", ""); + } + env.s("tensorOffsets",tensorOffsets.str()); env.s("kernelBody",body.str()); env.v("formals",formals); @@ -423,7 +549,7 @@ std::vector emitCompilationUnit(std::ostream & out, out << cpu_compilation_unit_template.format(env); } - return concat_desc; + return std::make_pair(std::move(concat_desc), has_random); } //////////////////////////////////////////////////////////////////////////////// @@ -518,7 +644,7 @@ void CompiledFusionFunction::launch_with_tensors(at::ArrayRef inputs char * buffer_next = buffer.data(); // A vector of arguments to the kernel. It's (numel, *input_descs, *output_descs) std::vector arguments; - arguments.reserve(1 + inputs.size() + flat_outputs_size); + arguments.reserve(3 + inputs.size() + flat_outputs_size); // Asserts that t's dims can be compressed in the same way as in desc // (that's what the kernel assumes), and appends it to the arguments vector. auto addTensorInfo = [&](TensorDesc & desc, const at::Tensor & t) { @@ -555,6 +681,19 @@ void CompiledFusionFunction::launch_with_tensors(at::ArrayRef inputs } } } + + // If the kernel call contains a random op, we need to pass in random seeds as + // well. + #ifdef USE_CUDA + if(has_random && this->backend() == at::kCUDA) { + auto gen_ = THCRandom_getGenerator(at::globalContext().getTHCState()); + uint64_t offset = + gen_->state.philox_seed_offset.fetch_add(this->get_rand_offset(numel)); + arguments.push_back(&gen_->state.initial_seed); + arguments.push_back(&offset); + } + #endif + launch_raw(numel, arguments.data()); } @@ -590,13 +729,15 @@ struct CUDAFusionFunction : public CompiledFusionFunction { checkCUDAVersion(prop); std::stringstream cu; - concat_desc = codegen::emitCompilationUnit(cu, name, agraph, true); + auto ret = codegen::emitCompilationUnit(cu, name, agraph, true); + concat_desc = std::move(ret.first); + has_random = ret.second; compilation_unit = cu.str(); nvrtcProgram program; TORCH_NVRTC_CHECK(nvrtcCreateProgram(&program, compilation_unit.c_str(), NULL, 0, nullptr, nullptr)); std::string compute = "--gpu-architecture=compute_" + std::to_string(prop.major) + std::to_string(prop.minor); - std::vector args = {"--std=c++11", compute.c_str()}; + std::vector args = {"--std=c++11", compute.c_str(), "-default-device"}; nvrtcResult result = nvrtcCompileProgram(program, args.size(), args.data()); if (result == NVRTC_ERROR_COMPILATION) { size_t logsize; @@ -630,8 +771,13 @@ struct CUDAFusionFunction : public CompiledFusionFunction { virtual at::Backend backend() const override { return at::kCUDA; } + virtual uint64_t get_rand_offset(uint32_t numel) override { + int numBlocks = std::min(maxBlocks, ceilDiv(numel, blockSize)); + return 4 * (ceil(numel/(4 * blockSize * numBlocks)) + 1); + } virtual void launch_raw(uint32_t numel, void ** arguments) override { int numBlocks = std::min(maxBlocks, ceilDiv(numel, blockSize)); + //std::cout << "maxBlocks = " << maxBlocks << " needed blocks: " << ceilDiv(numel,blockSize) // << " numblocks = " << numBlocks; @@ -786,7 +932,10 @@ struct CPUFusionFunction : public CompiledFusionFunction { TempFile cpp_file(cpp_template, 4); std::stringstream cu; - concat_desc = codegen::emitCompilationUnit(cu, name, agraph, false); + auto ret = codegen::emitCompilationUnit(cu, name, agraph, false); + concat_desc = std::move(ret.first); + has_random = ret.second; + JIT_ASSERT(!has_random); compilation_unit = cu.str(); cpp_file.write(compilation_unit); cpp_file.sync(); @@ -803,6 +952,9 @@ struct CPUFusionFunction : public CompiledFusionFunction { virtual at::Backend backend() const override { return at::kCPU; } + virtual uint64_t get_rand_offset(uint32_t numel) override { + return numel; + } virtual void launch_raw(uint32_t numel, void ** arguments) override { kernel(numel, arguments); } diff --git a/torch/csrc/jit/fusion_compiler.h b/torch/csrc/jit/fusion_compiler.h index c2f35ee0aa207..99a2303d37af2 100644 --- a/torch/csrc/jit/fusion_compiler.h +++ b/torch/csrc/jit/fusion_compiler.h @@ -5,7 +5,6 @@ #include #include "ATen/ATen.h" - #include #include #include @@ -109,6 +108,9 @@ struct CompiledFusionFunction { // launch_with_tensors handles packing at::Tensors into this arguments array. // CPU code uses the same convension so that launch_with_tensors can be shared. virtual void launch_raw(uint32_t numel, void ** arguments) = 0; + + virtual uint64_t get_rand_offset(uint32_t numel) = 0; + bool has_random; std::string name; // We keep these around for debugging std::string compilation_unit; diff --git a/torch/csrc/jit/passes/graph_fuser.cpp b/torch/csrc/jit/passes/graph_fuser.cpp index cc8dcb8926dee..071ca6d57cd2d 100644 --- a/torch/csrc/jit/passes/graph_fuser.cpp +++ b/torch/csrc/jit/passes/graph_fuser.cpp @@ -75,6 +75,7 @@ std::unordered_set simple_mappable = { // TODO support those //aten::clamp, //aten::lerp, + aten::rand_like, }; bool isSimpleMap(Node *node) { From cfa05706efa71454bede06c2aa25abc287db1f04 Mon Sep 17 00:00:00 2001 From: iotamudelta Date: Thu, 2 Aug 2018 09:02:03 -0700 Subject: [PATCH 02/16] ROCm contributions week 29 (#9653) Summary: In this changeset: * improvements to `hipify-python.py` * marking unit tests broken for ROCm * reducing the number of jobs for the built to avoid out of memory issues * switch to Thrust/cub-hip master for the CI Pull Request resolved: https://github.com/pytorch/pytorch/pull/9653 Differential Revision: D9117791 Pulled By: ezyang fbshipit-source-id: a6c3c7b81f2bda9825974bf9bf89a97767244352 --- .jenkins/pytorch/build.sh | 7 +++++ .jenkins/pytorch/test.sh | 4 +++ caffe2/CMakeLists.txt | 5 ++- cmake/public/LoadHIP.cmake | 3 ++ docker/caffe2/jenkins/common/install_rocm.sh | 1 - test/common.py | 1 + test/test_autograd.py | 5 ++- test/test_dataloader.py | 14 +++++++-- test/test_jit.py | 12 ++++++- test/test_optim.py | 3 +- test/test_utils.py | 4 ++- tools/amd_build/pyHIPIFY/hipify-python.py | 33 +++++++++++++++----- 12 files changed, 74 insertions(+), 18 deletions(-) diff --git a/.jenkins/pytorch/build.sh b/.jenkins/pytorch/build.sh index 28cde9e4a4fee..58db05af65f8b 100755 --- a/.jenkins/pytorch/build.sh +++ b/.jenkins/pytorch/build.sh @@ -31,10 +31,17 @@ pip install -r requirements.txt || true if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then export MAX_JOBS=4 + # This is necessary in order to cross compile (or else we'll have missing GPU device). export HCC_AMDGPU_TARGET=gfx900 + + # These environment variables are not set on CI when we were running as the Jenkins user. + # The HIP Utility scripts require these environment variables to be set in order to run without error. export LANG=C.UTF-8 export LC_ALL=C.UTF-8 + # This environment variable enabled HCC Optimizations that speed up the linking stage. + # https://github.com/RadeonOpenCompute/hcc#hcc-with-thinlto-linking + # export KMTHINLTO=1 python tools/amd_build/build_pytorch_amd.py USE_ROCM=1 python setup.py install --user exit 0 diff --git a/.jenkins/pytorch/test.sh b/.jenkins/pytorch/test.sh index 8443611f73224..7c8320b55e803 100755 --- a/.jenkins/pytorch/test.sh +++ b/.jenkins/pytorch/test.sh @@ -54,6 +54,10 @@ if [[ "$BUILD_ENVIRONMENT" == *asan* ]]; then (cd test && ! get_exit_code python -c "import torch; torch._C._crash_if_aten_asan(3)") fi +if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then + export PYTORCH_TEST_WITH_ROCM=1 +fi + if [[ "${JOB_BASE_NAME}" == *-NO_AVX-* ]]; then export ATEN_CPU_CAPABILITY=default elif [[ "${JOB_BASE_NAME}" == *-NO_AVX2-* ]]; then diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt index f345ee35f43ce..83e82a4f16e7e 100644 --- a/caffe2/CMakeLists.txt +++ b/caffe2/CMakeLists.txt @@ -355,9 +355,8 @@ if(USE_ROCM) target_include_directories(caffe2_hip PRIVATE ${Caffe2_HIP_INCLUDES}) target_include_directories(caffe2_hip INTERFACE $) - IF(BUILD_ATEN) - aten_set_target_props(caffe2_hip) - ENDIF() + # Set standard properties on the target + aten_set_target_props(caffe2_hip) # When a library has object files that contain device code, it needs to use hipcc/hcc to link. set_target_properties(caffe2_hip PROPERTIES LINKER_LANGUAGE HIP) diff --git a/cmake/public/LoadHIP.cmake b/cmake/public/LoadHIP.cmake index f70a5d171e3df..b3ee2ae9e72c6 100644 --- a/cmake/public/LoadHIP.cmake +++ b/cmake/public/LoadHIP.cmake @@ -102,6 +102,8 @@ FIND_PACKAGE(HIP 1.0) IF(HIP_FOUND) set(PYTORCH_FOUND_HIP TRUE) + ### Remove setting of Flags when FindHIP.CMake PR #558 is accepted.### + # https://github.com/ROCm-Developer-Tools/HIP/pull/558 # set(CMAKE_SHARED_LIBRARY_SONAME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG}) set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_PATH} -o " ) set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_PATH} -o -shared" ) @@ -109,6 +111,7 @@ IF(HIP_FOUND) set(CMAKE_HIP_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE}) set(CMAKE_HIP_ARCHIVE_APPEND ${CMAKE_CXX_ARCHIVE_APPEND}) set(CMAKE_HIP_ARCHIVE_FINISH ${CMAKE_CXX_ARCHIVE_FINISH}) + ### Remove setting of Flags when FindHIP.CMake PR #558 is accepted.### set(rocrand_DIR ${ROCRAND_PATH}/lib/cmake/rocrand) set(hiprand_DIR ${HIPRAND_PATH}/lib/cmake/hiprand) diff --git a/docker/caffe2/jenkins/common/install_rocm.sh b/docker/caffe2/jenkins/common/install_rocm.sh index 7a9e1f8eb59bc..9b69a917c3c2c 100644 --- a/docker/caffe2/jenkins/common/install_rocm.sh +++ b/docker/caffe2/jenkins/common/install_rocm.sh @@ -56,7 +56,6 @@ install_hip_thrust() { git clone --recursive https://github.com/ROCmSoftwarePlatform/Thrust.git /data/Thrust rm -rf /data/Thrust/thrust/system/cuda/detail/cub-hip git clone --recursive https://github.com/ROCmSoftwarePlatform/cub-hip.git /data/Thrust/thrust/system/cuda/detail/cub-hip - cd /data/Thrust/thrust/system/cuda/detail/cub-hip && git checkout hip_port_1.7.4_caffe2 && cd - } # This will be removed after merging an upcoming PR. diff --git a/test/common.py b/test/common.py index 7c4ec7b0eacc8..4dbe3c56c47c9 100644 --- a/test/common.py +++ b/test/common.py @@ -92,6 +92,7 @@ def _check_module_exists(name): NO_MULTIPROCESSING_SPAWN = os.environ.get('NO_MULTIPROCESSING_SPAWN', '0') == '1' TEST_WITH_ASAN = os.getenv('PYTORCH_TEST_WITH_ASAN', '0') == '1' TEST_WITH_UBSAN = os.getenv('PYTORCH_TEST_WITH_UBSAN', '0') == '1' +TEST_WITH_ROCM = os.getenv('PYTORCH_TEST_WITH_ROCM', '0') == '1' if TEST_NUMPY: import numpy diff --git a/test/test_autograd.py b/test/test_autograd.py index 7cba96d2e8a32..e99835002571e 100644 --- a/test/test_autograd.py +++ b/test/test_autograd.py @@ -15,7 +15,7 @@ from torch.autograd.function import once_differentiable from torch.autograd.profiler import profile from common import TEST_MKL, TestCase, run_tests, skipIfNoLapack, \ - suppress_warnings + suppress_warnings, TEST_WITH_ROCM from torch.autograd import Variable, Function, detect_anomaly from torch.autograd.function import InplaceFunction from torch.testing import make_non_contiguous, randn_like @@ -1574,6 +1574,7 @@ def test_pyscalar_conversions(self): self._test_pyscalar_conversions(lambda x: x.cuda(), lambda x: long(x)) @unittest.skipIf(not torch.cuda.is_available(), "CUDA unavailable") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_pin_memory(self): x = torch.randn(2, 2, requires_grad=True) self.assertEqual(x, x.pin_memory()) @@ -2389,6 +2390,7 @@ def f3(dt): f(dt) @unittest.skipIf(not torch.cuda.is_available(), "CUDA unavailable") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_set_requires_grad_only_for_floats_cuda(self): self._test_set_requires_grad_only_for_floats(self, True) @@ -2396,6 +2398,7 @@ def test_set_requires_grad_only_for_floats(self): self._test_set_requires_grad_only_for_floats(self, False) @unittest.skipIf(not torch.cuda.is_available(), "CUDA unavailable") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_rnn_backward_to_input_but_not_parameters_cuda(self): # this checks whether it is possible to not require # weight parameters, but require inputs, see #7722 diff --git a/test/test_dataloader.py b/test/test_dataloader.py index d1dabd266b878..bb61cced71753 100644 --- a/test/test_dataloader.py +++ b/test/test_dataloader.py @@ -13,7 +13,7 @@ from torch.utils.data import Dataset, TensorDataset, DataLoader, ConcatDataset from torch.utils.data.dataset import random_split from torch.utils.data.dataloader import default_collate, ExceptionWrapper, MANAGER_STATUS_CHECK_INTERVAL -from common import TestCase, run_tests, TEST_NUMPY, IS_WINDOWS, NO_MULTIPROCESSING_SPAWN +from common import TestCase, run_tests, TEST_NUMPY, IS_WINDOWS, NO_MULTIPROCESSING_SPAWN, TEST_WITH_ROCM # We cannot import TEST_CUDA from common_nn here, because if we do that, # the TEST_CUDNN line from common_nn will be executed multiple times @@ -335,12 +335,14 @@ def test_growing_dataset(self): self.assertEqual(len(dataloader_shuffle), 5) @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_sequential_pin_memory(self): loader = DataLoader(self.dataset, batch_size=2, pin_memory=True) for input, target in loader: self.assertTrue(input.is_pinned()) self.assertTrue(target.is_pinned()) + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_multiple_dataloaders(self): loader1_it = iter(DataLoader(self.dataset, num_workers=1)) loader2_it = iter(DataLoader(self.dataset, num_workers=2)) @@ -441,6 +443,7 @@ def test_batch_sampler(self): self._test_batch_sampler(num_workers=4) @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_shuffle_pin_memory(self): loader = DataLoader(self.dataset, batch_size=2, shuffle=True, num_workers=4, pin_memory=True) for input, target in loader: @@ -473,6 +476,7 @@ def test_error_workers(self): @unittest.skipIf(IS_WINDOWS, "FIXME: stuck test") @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_partial_workers(self): "check that workers exit even if the iterator is not exhausted" loader = iter(DataLoader(self.dataset, batch_size=2, num_workers=4, pin_memory=True)) @@ -526,8 +530,9 @@ def _is_process_alive(pid, pname): "spawn start method is not supported in Python 2, \ but we need it for creating another process with CUDA") @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") - def test_manager_unclean_exit(self): - '''there might be ConnectionResetError or leaked semaphore warning (due to dirty process exit), \ + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") + def test_main_process_unclean_exit(self): + '''There might be ConnectionResetError or leaked semaphore warning (due to dirty process exit), \ but they are all safe to ignore''' worker_pids = mp.Array('i', [0] * 4) @@ -629,6 +634,7 @@ def setUp(self): self.dataset = StringDataset() @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_shuffle_pin_memory(self): loader = DataLoader(self.dataset, batch_size=2, shuffle=True, num_workers=4, pin_memory=True) for batch_ndx, (s, n) in enumerate(loader): @@ -672,6 +678,7 @@ def test_sequential_batch(self): self.assertEqual(n[1], idx + 1) @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_pin_memory(self): loader = DataLoader(self.dataset, batch_size=2, pin_memory=True) for batch_ndx, sample in enumerate(loader): @@ -711,6 +718,7 @@ def _run_ind_worker_queue_test(self, batch_size, num_workers): if current_worker_idx == num_workers: current_worker_idx = 0 + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_ind_worker_queue(self): for batch_size in (8, 16, 32, 64): for num_workers in range(1, 6): diff --git a/test/test_jit.py b/test/test_jit.py index a2d18184947f9..bdac36e5cd802 100644 --- a/test/test_jit.py +++ b/test/test_jit.py @@ -9,7 +9,7 @@ from torch.autograd.function import traceable from torch.testing import assert_allclose from torch.onnx import OperatorExportTypes -from common import TestCase, run_tests, IS_WINDOWS, TEST_WITH_UBSAN +from common import TestCase, run_tests, IS_WINDOWS, TEST_WITH_UBSAN, TEST_WITH_ROCM from textwrap import dedent import os import io @@ -385,6 +385,7 @@ def forward(self, x): # TODO: Fuser doesn't work at all when inputs require grad. Fix that @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_lstm_fusion_cuda(self): inputs = get_lstm_inputs('cuda') ge = self.checkTrace(LSTMCellF, inputs) @@ -408,6 +409,7 @@ def test_lstm_fusion_cpu(self): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_lstm_fusion_concat(self): inputs = get_lstm_inputs('cuda') ge = self.checkTrace(LSTMCellC, inputs) @@ -415,6 +417,7 @@ def test_lstm_fusion_concat(self): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_concat_fusion(self): hx = torch.randn(3, 20, dtype=torch.float, device='cuda') cx = torch.randn(3, 20, dtype=torch.float, device='cuda') @@ -427,6 +430,7 @@ def foo(hx, cx): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_fusion_distribute(self): def f(x, y): z1, z2 = (x + y).chunk(2, dim=1) @@ -471,6 +475,7 @@ def fn_test_comparison_gt_lt(x, y): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_comparison_gt_lt(self): x = torch.randn(4, 4, dtype=torch.float, device='cuda') y = torch.randn(4, 4, dtype=torch.float, device='cuda') @@ -479,6 +484,7 @@ def test_comparison_gt_lt(self): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_comparison_ge_le(self): def f(x, y): mask = (x >= 0).type_as(x) @@ -498,6 +504,7 @@ def fn_test_relu(x, y): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_relu(self): x = torch.randn(4, 4, dtype=torch.float, device='cuda') y = torch.randn(4, 4, dtype=torch.float, device='cuda') @@ -520,6 +527,7 @@ def fn_test_exp(x, y): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_exp(self): x = torch.randn(4, 4, dtype=torch.float, device='cuda') y = torch.randn(4, 4, dtype=torch.float, device='cuda') @@ -864,6 +872,7 @@ def doit(x, y): @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") @unittest.skipIf(not RUN_CUDA, "cpp tests require CUDA") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_cpp(self): # rather than rebuild assertExpected in cpp, # just glob all the cpp outputs into one file for now @@ -2560,6 +2569,7 @@ def test_tensor_number_math(self): self._test_tensor_number_math() @unittest.skipIf(not RUN_CUDA, "No CUDA") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_tensor_number_math_cuda(self): self._test_tensor_number_math(device='cuda') diff --git a/test/test_optim.py b/test/test_optim.py index 8775ef0b55f8d..2d5b876dd3a8e 100644 --- a/test/test_optim.py +++ b/test/test_optim.py @@ -11,7 +11,7 @@ from torch.autograd import Variable from torch import sparse from torch.optim.lr_scheduler import LambdaLR, StepLR, MultiStepLR, ExponentialLR, CosineAnnealingLR, ReduceLROnPlateau -from common import TestCase, run_tests, TEST_WITH_UBSAN +from common import TestCase, run_tests, TEST_WITH_UBSAN, TEST_WITH_ROCM def rosenbrock(tensor): @@ -437,6 +437,7 @@ def test_asgd(self): with self.assertRaisesRegex(ValueError, "Invalid weight_decay value: -0.5"): optim.ASGD(None, lr=1e-2, weight_decay=-0.5) + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_rprop(self): self._test_rosenbrock( lambda params: optim.Rprop(params, lr=1e-3), diff --git a/test/test_utils.py b/test/test_utils.py index 077f789a0a142..c6559fe68fae8 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -19,7 +19,7 @@ from torch.utils.trainer.plugins.plugin import Plugin from torch.autograd._functions.utils import prepare_onnx_paddings from torch.autograd._functions.utils import check_onnx_broadcast -from common import IS_WINDOWS, IS_PPC +from common import IS_WINDOWS, IS_PPC, TEST_WITH_ROCM HAS_CUDA = torch.cuda.is_available() @@ -412,6 +412,7 @@ def test_cpu(self): @unittest.skipIf(not HAS_CFFI or not HAS_CUDA, "ffi tests require cffi package") @unittest.skipIf(IS_WINDOWS, "ffi doesn't currently work on Windows") + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_gpu(self): create_extension( name='gpulib', @@ -615,6 +616,7 @@ def test_bottleneck_cpu_only(self): self._check_cuda(out) @unittest.skipIf(not HAS_CUDA, 'No CUDA') + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_bottleneck_cuda(self): rc, out, err = self._run_bottleneck('bottleneck/test_cuda.py') self.assertEqual(rc, 0, 'Run failed with\n{}'.format(err)) diff --git a/tools/amd_build/pyHIPIFY/hipify-python.py b/tools/amd_build/pyHIPIFY/hipify-python.py index fc3efabd26db7..355948433ea7c 100755 --- a/tools/amd_build/pyHIPIFY/hipify-python.py +++ b/tools/amd_build/pyHIPIFY/hipify-python.py @@ -465,7 +465,7 @@ def disable_asserts(input_string): def replace_forceinline(input_string): - """__forceinline__'d methods can cause 'symbol multiply defined' errors in HIP. + """__forceinline__'d methods can cause 'symbol multiply defined' errors in HIP. Adding 'static' to all such methods leads to compilation errors, so replacing '__forceinline__' with 'inline' as a workaround https://github.com/ROCm-Developer-Tools/HIP/blob/master/docs/markdown/hip_faq.md#what-if-hip-generates-error-of-symbol-multiply-defined-only-on-amd-machine @@ -487,6 +487,22 @@ def replace_math_functions(input_string): return output_string +def replace_extern_shared(input_string): + """Match extern __shared__ type foo[]; syntax and use HIP_DYNAMIC_SHARED() MACRO instead. + https://github.com/ROCm-Developer-Tools/HIP/blob/master/docs/markdown/hip_kernel_language.md#__shared__ + Example: + "extern __shared__ char smemChar[];" => "HIP_DYNAMIC_SHARED( char, smemChar)" + "extern __shared__ unsigned char smem[];" => "HIP_DYNAMIC_SHARED( unsigned char, my_smem)" + """ + output_string = input_string + output_string = re.sub( + r"extern\s+([\w\(\)]+)?\s*__shared__\s+([\w:<>\s]+)\s+(\w+)\s*\[\s*\]\s*;", + lambda inp: "HIP_DYNAMIC_SHARED({0} {1}, {2})".format( + inp.group(1) or "", inp.group(2), inp.group(3)), output_string) + + return output_string + + def disable_function(input_string, function, replace_style): """ Finds and disables a function in a particular file. @@ -681,9 +697,9 @@ def preprocessor(filepath, stats, hipify_caffe2): if cuda_type in output_source: if hipify_caffe2: - pattern = r'({0})'.format(cuda_type) + pattern = r'({0})'.format(re.escape(cuda_type)) else: - pattern = r'(\b{0}\b)'.format(cuda_type) + pattern = r'(\b{0}\b)'.format(re.escape(cuda_type)) output_source = re.sub(pattern, hip_type, output_source) # Perform Kernel Launch Replacements @@ -699,6 +715,9 @@ def preprocessor(filepath, stats, hipify_caffe2): # Replace __forceinline__ with inline output_source = replace_forceinline(output_source) + # Replace the extern __shared__ + output_source = replace_extern_shared(output_source) + fout.write(output_source) @@ -706,7 +725,7 @@ def file_specific_replacement(filepath, search_string, replace_string, strict=Fa with openf(filepath, "r+") as f: contents = f.read() if strict: - contents = re.sub(r'\b({0})\b'.format(search_string), lambda x: replace_string, contents) + contents = re.sub(r'\b({0})\b'.format(re.escape(search_string)), lambda x: replace_string, contents) else: contents = contents.replace(search_string, replace_string) f.seek(0) @@ -824,7 +843,7 @@ def disable_unsupported_function_call(function, input_string, replacement): output_string = input_string # Find all calls to the function - calls = re.finditer(r"\b{0}\b".format(function), input_string) + calls = re.finditer(r"\b{0}\b".format(re.escape(function)), input_string) # Do replacements for call in calls: @@ -983,7 +1002,7 @@ def replace_arg(match): if "THCUNN" in filepath.split("/") and "generic" not in filepath.split("/"): kernel_name_with_template = kernel_name_with_template.replace("", "") - full_new_kernel_launch = re.sub(r'\b{0}\b'.format(original_kernel_name_with_template), + full_new_kernel_launch = re.sub(r'\b{0}\b'.format(re.escape(original_kernel_name_with_template)), lambda x: kernel_name_with_template, full_new_kernel_launch) # Replace Launch @@ -1181,7 +1200,7 @@ def main(): # Disable Constants w\ Boundary. for const in constants: - txt = re.sub(r"\b{0}\b".format(const), constants[const], txt) + txt = re.sub(r"\b{0}\b".format(re.escape(const)), constants[const], txt) # Disable Constants for s_const in s_constants: From 9c818bfbc7d91961c580c32d7f29dbbaf85aaf2f Mon Sep 17 00:00:00 2001 From: James Reed Date: Thu, 2 Aug 2018 10:13:42 -0700 Subject: [PATCH 03/16] Refactor PythonValue types + use tryMatchSchema for PythonOp Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/10132 Differential Revision: D9121327 Pulled By: jamesr66a fbshipit-source-id: 6d8bcf6b0dca54106cf9ed740bcff857062a03da --- test/test_jit.py | 28 +-- torch/csrc/jit/script/compiler.cpp | 9 +- torch/csrc/jit/script/compiler.h | 3 +- torch/csrc/jit/script/init.cpp | 296 ++++++++++++++++------------- 4 files changed, 177 insertions(+), 159 deletions(-) diff --git a/test/test_jit.py b/test/test_jit.py index bdac36e5cd802..afcfb85d723c7 100644 --- a/test/test_jit.py +++ b/test/test_jit.py @@ -3395,7 +3395,8 @@ def fn(x : torch.Tensor, y : Tensor, z) -> Tuple[Tensor, Tensor, Tensor]: f.write(code) fn = get_fn('test_type_annotation_py3', script_path) - with self.assertRaisesRegex(RuntimeError, r"expected Tensor, but got"): + with self.assertRaisesRegex(RuntimeError, r"expected a value of type Tensor for argument" + r" '0' but found \(Tensor, Tensor\)"): @torch.jit.script def bad_fn(x): x, y = fn((x, x), x, x) @@ -3454,13 +3455,13 @@ def method(self, x): y = self.baz(x) return x - with self.assertRaisesRegex(RuntimeError, "incorrect number of arguments: expected 1, but got 2"): + with self.assertRaisesRegex(RuntimeError, "expected at most 1 arguments but found 2"): ModuleTooMany() - with self.assertRaisesRegex(RuntimeError, "incorrect number of arguments: expected 2, but got 1"): + with self.assertRaisesRegex(RuntimeError, "argument 1 not provided"): ModuleTooFew() with self.assertRaisesRegex(RuntimeError, "need 3 values .* found only 2"): ModuleTooManyAssign() - with self.assertRaisesRegex(RuntimeError, "incorrect number of arguments: expected 2, but got 1"): + with self.assertRaisesRegex(RuntimeError, "argument 1 not provided."): ModuleDefault() def test_script_define_order(self): @@ -4265,18 +4266,6 @@ def test_wrong_use_as_callable(self): def wrong_use_as_callable(x): return x(3, 4, 5) - def test_wrong_python_kwarg_call(self): - with self.assertRaisesRegex(RuntimeError, 'keyword arguments in Python calls aren\'t supported'): - # NB: the only way I could get to this code path is if I made the - # python function have 0 inputs, since we interpret all the inputs to - # the function as inputs (including those with defaults) and not kwargs - def test_fn(): - return 3 - - @torch.jit.script - def wrong_python_kwarg_call(self): - return test_fn(attr=6) - def test_python_val_doesnt_have_attr(self): with self.assertRaisesRegex(RuntimeError, 'object has no attribute abcd'): def test_fn(): @@ -4287,7 +4276,7 @@ def python_val_doesnt_have_attr(): return test_fn.abcd def test_wrong_module_attr_lookup(self): - with self.assertRaisesRegex(RuntimeError, 'unsupported attribute lookup on'): + with self.assertRaisesRegex(RuntimeError, 'python value of type \'type\' cannot be used as a value:'): import io @torch.jit.script @@ -4295,7 +4284,7 @@ def wrong_module_attr_lookup(): return io.BytesIO def test_wrong_method_call_inputs(self): - with self.assertRaisesRegex(RuntimeError, 'argument \'y\' not provided'): + with self.assertRaisesRegex(RuntimeError, 'argument y not provided'): class SomeModule(torch.jit.ScriptModule): @torch.jit.script_method @@ -4870,7 +4859,8 @@ def to_inline(x, y): def some_func(x): return to_inline((x, x), x) - some_func(torch.rand(3, 4)) + x = torch.rand(3, 4) + self.assertEqual(some_func(x), x) def test_file_format_serialization(self): import tempfile diff --git a/torch/csrc/jit/script/compiler.cpp b/torch/csrc/jit/script/compiler.cpp index 4f27cb25b53cb..9e3c3f7ea5f74 100644 --- a/torch/csrc/jit/script/compiler.cpp +++ b/torch/csrc/jit/script/compiler.cpp @@ -189,8 +189,9 @@ struct Environment { SugaredValuePtr createCapturedInputIfNeeded(const SourceRange& loc, std::string ident) { auto in_frame = findInThisFrame(ident); - if (in_frame) + if (in_frame) { return in_frame; + } // recursively handles the case where parent blocks are also loops auto from_parent = next ? next->createCapturedInputIfNeeded(loc, ident) : nullptr; @@ -266,7 +267,7 @@ struct Environment { auto retval = createCapturedInputIfNeeded(range, ident); if(!retval) { - retval = resolver(ident); + retval = resolver(ident, method, range); } if(!retval) { @@ -417,7 +418,7 @@ at::optional> tryMatchSchema( return at::nullopt; } if(positional_inputs[*idx]) { - err() << "argument '" << nv.name << "' specified twice \n" << nv.loc; + err() << "argument " << nv.name << " specified twice \n" << nv.loc; return at::nullopt; } positional_inputs[*idx] = nv; @@ -428,7 +429,7 @@ at::optional> tryMatchSchema( continue; auto default_value = schema.arguments[i].default_value; if(!default_value) { - err() << "argument '" << schema.arguments[i].name << "' not provided.\n" << loc; + err() << "argument " << schema.arguments[i].name << " not provided.\n" << loc; return at::nullopt; } positional_inputs[i] = NamedValue( diff --git a/torch/csrc/jit/script/compiler.h b/torch/csrc/jit/script/compiler.h index 3c4dcb07a248e..c5175815144e1 100644 --- a/torch/csrc/jit/script/compiler.h +++ b/torch/csrc/jit/script/compiler.h @@ -131,7 +131,8 @@ struct TORCH_API BuiltinFunction : public SugaredValue { size_t n_binders) override; }; -using Resolver = std::function(const std::string& name)>; +using Resolver = std::function(const std::string& name, Method& m, const SourceRange& loc)>; TORCH_API void defineMethodsInModule( Module & m, const std::vector& definitions, diff --git a/torch/csrc/jit/script/init.cpp b/torch/csrc/jit/script/init.cpp index 39bb51ed89ca5..b91d348ed627d 100644 --- a/torch/csrc/jit/script/init.cpp +++ b/torch/csrc/jit/script/init.cpp @@ -45,18 +45,40 @@ inline std::shared_ptr toSimple(Value* v) { return std::make_shared(v); } +// NB: This should be the single entry-point for instantiating a SugaredValue +// from a Python object. If you are adding support for converting a new Python +// type, *add it in this function's implementation*. +std::shared_ptr toSugaredValue( + py::object obj, + Method& m, + SourceRange loc, + bool is_constant = false, + bool is_submodule = false); + struct VISIBILITY_HIDDEN PythonValue : public SugaredValue { PythonValue(py::object self) : self(std::move(self)) {} - std::tuple, std::vector> getArgumentTypes(const size_t n_args, const size_t n_binders) { + FunctionSchema getSchema(const size_t n_args, const size_t n_binders) { auto annotations = py::module::import("torch.jit.annotations"); auto signature = annotations.attr("get_signature")(self); + std::vector args, rets; // We may mutate this if we can determine the number of args from Python // introspection. size_t actual_n_args = n_args; if (!signature.is_none()) { - return py::cast, std::vector>>(signature); + std::vector arg_types, ret_types; + std::tie(arg_types, ret_types) = py::cast, std::vector>>(signature); + args.reserve(arg_types.size()); + size_t idx = 0; // Fake argument names by putting in the index + for (auto &arg_type : arg_types) { + args.push_back(Argument(std::to_string(idx++), std::move(arg_type), {}, {}, false)); + } + rets.reserve(ret_types.size()); + idx = 0; + for (auto &ret_type : ret_types) { + rets.push_back(Argument(std::to_string(idx++), std::move(ret_type), {}, {}, false)); + } } else { // Create a default signature using what information we have @@ -71,62 +93,36 @@ struct VISIBILITY_HIDDEN PythonValue : public SugaredValue { } // Construct the default signature: all arguments and returns will be // DynamicType - return std::make_pair(std::vector(actual_n_args, DynamicType::get()), std::vector(n_binders, DynamicType::get())); + args.reserve(actual_n_args); + for (size_t i=0; i < actual_n_args; ++i) { + args.push_back(Argument(std::to_string(i), DynamicType::get(), {}, {}, false)); + } + rets.reserve(n_binders); + for (size_t i = 0; i < n_binders; ++i) { + rets.push_back(Argument(std::to_string(i), DynamicType::get(), {}, {}, false)); + } } + return FunctionSchema("", std::move(args), std::move(rets)); } // call it like a function, e.g. `outputs = this(inputs)` virtual std::shared_ptr call(SourceRange loc, Method & m, at::ArrayRef inputs_, at::ArrayRef attributes, size_t n_binders) override { auto inputs = toValues(inputs_); - std::vector arguments, returns; - std::tie(arguments, returns) = getArgumentTypes(inputs.size(), n_binders); - - if (arguments.size() != inputs.size()) - throw ErrorReport(loc) << "calling a Python function with an incorrect number " - << "of arguments: expected " << arguments.size() << ", but got " - << inputs.size(); - for (size_t i = 0; i < arguments.size(); ++i) { - if (!inputs[i]->type()->isSubtypeOf(arguments[i])) - throw ErrorReport(loc) << "type mismatch at argument " << i << ": expected " - << arguments[i]->str() << ", but got " << inputs[i]->type()->str(); - } - // We have to do this check here, because implementation of this function is tightly - // coupled with the impl for PythonOp in the interpreter. Right now it assumes that - // all inputs taken from the stack are Tensors, so that's what we have to do. - ensureTensors(loc, inputs); - - if (attributes.size() > 0) - throw ErrorReport(loc) << "keyword arguments in Python calls aren't supported"; - Graph& g = *m.graph(); - - // this python object might be a @trace or @script function/module - // if so, inline the graph rather than calling the python - - if(py::isinstance(self)) { - Module& mod = py::cast(self); - if (Method * forward = mod.find_method("forward")) { - // This code path should only get called for Modules that are really - // wrappers around pure script/traced functions. Modules with parameters - // should be submodules of the caller, and thus will be represented as - // ModuleValue and not go through here. - if (mod.get_parameters().size() != 0) { - throw ErrorReport(loc) << "Attempted to inline a Module with parameters. " - "Stateful modules to be inlined must be submodules of the callee."; - } - std::vector named_inputs; - for (auto inp : inputs) - named_inputs.push_back(NamedValue(loc, "", inp)); - return packOutputs(*m.graph(), m.emit_call_to(loc, *forward, named_inputs, {})); - } - } + auto schema = getSchema(inputs.size(), n_binders); + + std::stringstream failure_messages; + at::optional> all_inputs = + tryMatchSchema(schema, loc, *m.graph(), inputs_, attributes, failure_messages); + if (!all_inputs) + throw ErrorReport(loc) << failure_messages.str(); // Release the function object so we can wrap it in a PythonOp py::object func = self; std::string cconv(inputs.size(), 't'); - Node* new_node = g.insertNode(g.createPythonOp( + Node* new_node = m.graph()->insertNode(m.graph()->createPythonOp( THPObjectPtr(func.release().ptr()), cconv, {})); new_node->setSourceLocation(std::make_shared(loc)); - for(auto i : inputs) + for(auto &i : *all_inputs) new_node->addInput(i); // This is really dumb, but relaxing the constraints on return types would @@ -134,14 +130,14 @@ struct VISIBILITY_HIDDEN PythonValue : public SugaredValue { // Note that this effectively makes the return type of Tuple[Tensor] and Tensor // equivalent, but the PythonOp impl ends with an optional tuple unpack, so we need // to do it. - for (auto & ret_type_elem : returns) { - if (!ret_type_elem->isSubtypeOf(DynamicType::get())) { + for (auto & ret_arg : schema.returns) { + if (!ret_arg.type->isSubtypeOf(DynamicType::get())) { throw ErrorReport(loc) << "Python functions can currently only return Tensors"; } } std::vector outputs; - for(size_t i = 0; i < returns.size(); ++i) + for(size_t i = 0; i < schema.returns.size(); ++i) outputs.push_back(new_node->addOutput()); return packOutputs(*m.graph(), outputs); } @@ -155,13 +151,6 @@ struct VISIBILITY_HIDDEN PythonValue : public SugaredValue { } protected: - bool isBuiltinModule() { - // XXX: these can't be static, or they will be destructed after the Python interpreter - // exits and that generally sounds like a bad idea - py::object torch = py::module::import("torch"); - py::object functional = py::module::import("torch.nn.functional"); - return self.is(torch) || self.is(functional); - } py::object getattr(SourceRange loc, const std::string& name) { try { @@ -174,56 +163,48 @@ struct VISIBILITY_HIDDEN PythonValue : public SugaredValue { py::object self; }; -// by using torch.jit.Const, a user can mark a python value constant -// we then make that value immutable. -// once marked constant, we enable additional behavior such as -// 1. conversion via asValue to a constant Tensor -// 2. unrolling of for loops -struct VISIBILITY_HIDDEN ConstantPythonValue : public PythonValue { - using PythonValue::PythonValue; - virtual Value * asValue(SourceRange loc, Method & m) override { +struct VISIBILITY_HIDDEN PythonModuleValue : public PythonValue { + explicit PythonModuleValue(py::object mod) : PythonValue(mod) {} - return PythonValue::asValue(loc, m); + std::shared_ptr attr(SourceRange loc, Method & m, const std::string& field) override { + py::object member = getattr(loc, field); + return toSugaredValue(member, m, loc); } - virtual std::vector> asTuple(SourceRange loc, Method& m) override { - if(!py::isinstance(self)) - return PythonValue::asTuple(loc, m); + private: +}; + +struct VISIBILITY_HIDDEN BuiltinPythonModuleValue : public PythonModuleValue { + explicit BuiltinPythonModuleValue(py::object mod) : PythonModuleValue(mod) {} + std::shared_ptr attr(SourceRange loc, Method & m, const std::string& field) override { + // We support calling functions and using type/layout/device constants + // on the torch builtin modules + py::object member = getattr(loc, field); + if (py::isinstance(member)) { + return std::make_shared(field, at::nullopt); + } + return toSugaredValue(member, m, loc, /*is_constant =*/true); + } +}; +bool isBuiltinModule(py::object obj) { + // XXX: these can't be static, or they will be destructed after the Python interpreter + // exits and that generally sounds like a bad idea + py::object torch = py::module::import("torch"); + py::object functional = py::module::import("torch.nn.functional"); + return obj.is(torch) || obj.is(functional); +} + +struct VISIBILITY_HIDDEN ConstantPythonTupleValue : public PythonValue { + explicit ConstantPythonTupleValue(py::object tup) : PythonValue(tup) {} + std::vector> asTuple(SourceRange loc, Method& m) override { py::tuple tup = self; std::vector> result; - for(size_t i = 0; i < tup.size(); ++i) { - result.push_back(create(loc, m, tup[i])); + result.reserve(tup.size()); + for (size_t i = 0; i < tup.size(); ++i) { + result.push_back(toSugaredValue(tup[i], m, loc, true)); } return result; } - static std::shared_ptr create(SourceRange loc, Method& m, py::object self) { - // directly create SimpleValues when possible, because they are first-class - // and can be re-assigned. Otherwise, this would be invalid: - // f = python_constant - // while ... - // f = f + 1 - auto& g = *m.graph(); - if(py::isinstance(self)) { - return toSimple(insertConstant(g, py::cast(self), loc)); - } else if(py::isinstance(self)) { - return toSimple(insertConstant(g, py::cast(self), loc)); - } else if(py::isinstance(self)) { - return toSimple(insertConstant(g, py::cast(self), loc)); - } else if(THPDevice_Check(self.ptr())) { - auto device = (THPDevice*) self.ptr(); - std::vector v = {static_cast(device->device.type()), device->device.index()}; - return toSimple(insertConstant(g, std::move(v))); - } else if(THPLayout_Check(self.ptr())) { - auto layout = (THPLayout*) self.ptr(); - const auto v = static_cast(layout->layout); - return toSimple(insertConstant(g, v, loc)); - } else if(THPDtype_Check(self.ptr())) { - auto dtype = (THPDtype*)(self.ptr()); - const auto v = static_cast(dtype->scalar_type); - return toSimple(insertConstant(g, v, loc)); - } - return std::make_shared(self); - } }; std::shared_ptr PythonValue::attr(SourceRange loc, Method & m, const std::string& field) { @@ -231,32 +212,8 @@ std::shared_ptr PythonValue::attr(SourceRange loc, Method & m, con // make an exception for traversing modules because we want to be access // torch, torch.nn.functional, and the functions they expose. py::object member = getattr(loc, field); - if (isBuiltinModule()) { - if(py::isinstance(member)) { - return std::make_shared(field, at::nullopt); - } - //e.g. any tensor attribute objects such as torch.uint8 - if(THPDtype_Check(member.ptr()) || - THPLayout_Check(member.ptr()) || - THPDevice_Check(member.ptr())) { - return ConstantPythonValue::create(loc, m, member); - } - } - if (py::isinstance(self) && py::isinstance(member)) { - return std::make_shared(member); - } - throw ErrorReport(loc) << "unsupported attribute lookup on " << py::repr(self) << "."; -} -Resolver pythonResolver(ResolutionCallback rcb) { - return [=](const std::string& name) -> std::shared_ptr { - AutoGIL ag; - py::object obj = rcb(name); - if(obj.is(py::none())) { - return nullptr; - } - return std::make_shared(obj); - }; + return toSugaredValue(member, m, loc); } // defines how modules/methods behave inside the script subset. @@ -305,11 +262,10 @@ struct ModuleValue : public SugaredValue { // python method. If so return this as a python value. py::object py_module = py::cast(module); if(py::object attr = py::getattr(py_module, field.c_str(), py::none())) { - if(py::isinstance(attr) || - py::isinstance(attr, py::module::import("torch.nn").attr("Module"))) { - return std::make_shared(attr); - } else if(py_module.attr("_constants_set").contains(field.c_str())) { - return ConstantPythonValue::create(loc, m, attr); + if (py::isinstance(attr) || + py::isinstance(attr, py::module::import("torch.nn").attr("Module")) || + py_module.attr("_constants_set").contains(field.c_str())) { + return toSugaredValue(attr, m, loc, true); } else { throw ErrorReport(loc) << "attribute '" << field << "' of type '" << typeString(attr) << "' is not usable in a script method (did you forget to add it __constants__?)"; } @@ -329,20 +285,77 @@ struct ModuleValue : public SugaredValue { std::vector> result; for(py::handle module : py_module) { py::object obj = py::reinterpret_borrow(module); - if(py::isinstance(obj)) { - auto r = py::cast>(obj); - result.push_back(std::make_shared(r)); - } else { - result.push_back(ConstantPythonValue::create(loc, m, obj)); - } + result.push_back(toSugaredValue( + obj, + m, + loc, + /*is_constant =*/false, + /*is_submodule =*/true)); } return result; } -private: + private: std::shared_ptr module; }; +std::shared_ptr toSugaredValue( + py::object obj, + Method& m, + SourceRange loc, + bool is_constant, + bool is_submodule) { + // directly create SimpleValues when possible, because they are first-class + // and can be re-assigned. Otherwise, this would be invalid: + // f = python_constant + // while ... + // f = f + 1 + auto& g = *m.graph(); + if (is_constant) { + if (py::isinstance(obj)) { + return toSimple(insertConstant(g, py::cast(obj), loc)); + } else if (py::isinstance(obj)) { + return toSimple(insertConstant(g, py::cast(obj), loc)); + } else if (py::isinstance(obj)) { + return toSimple(insertConstant(g, py::cast(obj), loc)); + } else if (THPDevice_Check(obj.ptr())) { + auto device = (THPDevice*)obj.ptr(); + std::vector v = {static_cast(device->device.type()), + device->device.index()}; + return toSimple(insertConstant(g, std::move(v))); + } else if (THPLayout_Check(obj.ptr())) { + auto layout = (THPLayout*)obj.ptr(); + const auto v = static_cast(layout->layout); + return toSimple(insertConstant(g, v, loc)); + } else if (THPDtype_Check(obj.ptr())) { + auto dtype = (THPDtype*)(obj.ptr()); + const auto v = static_cast(dtype->scalar_type); + return toSimple(insertConstant(g, v, loc)); + } else if (py::isinstance(obj)) { + return std::make_shared(obj); + } + } + if (py::isinstance(obj)) { + auto mod = py::cast>(obj); + // In the case that this Python object is not a submodule, inline *ONLY + // PURE* ScriptModules. This allows us to call arbitrary @script functions + // within a scripting context while still enforcing that parameters from + // stateful submodules are properly accounted for. + if (!is_submodule && mod->get_parameters().size() != 0) { + throw ErrorReport() + << "Attempted to inline a Module with parameters. " + "Stateful modules to be inlined must be submodules of the callee."; + } + return std::make_shared(mod); + } else if (py::isinstance(obj)) { + if (isBuiltinModule(obj)) { + return std::make_shared(obj); + } else { + return std::make_shared(obj); + } + } + return std::make_shared(obj); +} py::object unpackVariableTensorList(std::vector outputs) { // if we don't tell pybind these are variables it chokes on the @@ -381,6 +394,19 @@ py::object runMethodFromPython(Method& m, py::args args) { return wrapStack(std::move(stack), m.graph()->outputs()); } +Resolver pythonResolver(ResolutionCallback rcb) { + return [=](const std::string& name, + Method& m, + const SourceRange& loc) -> std::shared_ptr { + AutoGIL ag; + py::object obj = rcb(name); + if (obj.is(py::none())) { + return nullptr; + } + return toSugaredValue(obj, m, loc); + }; +} + void initJitScriptBindings(PyObject* module) { auto m = py::handle(module).cast(); From 230ca98d4bc2670ba43451804277b7736a890cb8 Mon Sep 17 00:00:00 2001 From: Gregory Chanan Date: Thu, 2 Aug 2018 10:28:45 -0700 Subject: [PATCH 04/16] Remove THTensor_isSize. (#10146) Summary: This is part of the process of removing THLongStorage to represent sizes/strides. Pull Request resolved: https://github.com/pytorch/pytorch/pull/10146 Differential Revision: D9126611 Pulled By: gchanan fbshipit-source-id: b0d995a4c51dfd54bf76dcfee9a69f37f9d01652 --- aten/src/TH/generic/THTensor.cpp | 14 -------------- aten/src/TH/generic/THTensor.h | 1 - aten/src/THC/generic/THCTensor.cpp | 14 -------------- aten/src/THC/generic/THCTensor.h | 1 - aten/src/THC/generic/THCTensorScatterGather.cu | 4 +--- aten/src/THC/generic/THCTensorSort.cu | 4 +--- aten/src/THCUNN/common.h | 14 +++++--------- aten/src/THNN/THNN.h | 8 -------- aten/src/THNN/generic/MultiLabelMarginCriterion.c | 4 +++- aten/src/THNN/init.cpp | 6 +----- 10 files changed, 11 insertions(+), 59 deletions(-) diff --git a/aten/src/TH/generic/THTensor.cpp b/aten/src/TH/generic/THTensor.cpp index e68c60a9455c4..20ddd70475119 100644 --- a/aten/src/TH/generic/THTensor.cpp +++ b/aten/src/TH/generic/THTensor.cpp @@ -596,20 +596,6 @@ int THTensor_(isContiguous)(const THTensor *self) return 1; } -int THTensor_(isSize)(const THTensor *self, const THLongStorage *dims) -{ - int d; - if (THTensor_nDimensionLegacyAll(self) != dims->size) - return 0; - - for(d = 0; d < THTensor_nDimensionLegacyAll(self); ++d) - { - if(self->size(d) != THLongStorage_data(dims)[d]) - return 0; - } - return 1; -} - int THTensor_(isSameSizeAs)(const THTensor *self, const THTensor* src) { int d; diff --git a/aten/src/TH/generic/THTensor.h b/aten/src/TH/generic/THTensor.h index 126fc504dee48..664fdd5a89f26 100644 --- a/aten/src/TH/generic/THTensor.h +++ b/aten/src/TH/generic/THTensor.h @@ -113,7 +113,6 @@ TH_API void THTensor_(unsqueeze1d)(THTensor *self, THTensor *src, int dimension_ TH_API int THTensor_(isContiguous)(const THTensor *self); TH_API int THTensor_(isSameSizeAs)(const THTensor *self, const THTensor *src); TH_API int THTensor_(isSetTo)(const THTensor *self, const THTensor *src); -TH_API int THTensor_(isSize)(const THTensor *self, const THLongStorage *dims); TH_API ptrdiff_t THTensor_(nElement)(const THTensor *self); TH_API void THTensor_(retain)(THTensor *self); diff --git a/aten/src/THC/generic/THCTensor.cpp b/aten/src/THC/generic/THCTensor.cpp index 940af6eb86ead..83dfe36c54fcc 100644 --- a/aten/src/THC/generic/THCTensor.cpp +++ b/aten/src/THC/generic/THCTensor.cpp @@ -523,20 +523,6 @@ int THCTensor_(isContiguous)(THCState *state, const THCTensor *self) return THCTensor_isContiguous(state, self); } -int THCTensor_(isSize)(THCState *state, const THCTensor *self, const THLongStorage *dims) -{ - int d; - if (self->dim() != dims->size) - return 0; - - for (d = 0; d < self->dim(); ++d) - { - if (self->size(d) != THLongStorage_data(dims)[d]) - return 0; - } - return 1; -} - int THCTensor_(isSetTo)(THCState *state, const THCTensor *self, const THCTensor *src) { if (THTensor_getStoragePtr(self) == THTensor_getStoragePtr(src) && diff --git a/aten/src/THC/generic/THCTensor.h b/aten/src/THC/generic/THCTensor.h index 2ee1bf11a4be4..3f6b94eb4ba52 100644 --- a/aten/src/THC/generic/THCTensor.h +++ b/aten/src/THC/generic/THCTensor.h @@ -115,7 +115,6 @@ THC_API void THCTensor_(unsqueeze1d)(THCState *state, THCTensor *self, THCTensor THC_API int THCTensor_(isContiguous)(THCState *state, const THCTensor *self); THC_API int THCTensor_(isSameSizeAs)(THCState *state, const THCTensor *self, const THCTensor *src); THC_API int THCTensor_(isSetTo)(THCState *state, const THCTensor *self, const THCTensor *src); -THC_API int THCTensor_(isSize)(THCState *state, const THCTensor *self, const THLongStorage *dims); THC_API ptrdiff_t THCTensor_(nElement)(THCState *state, const THCTensor *self); THC_API void THCTensor_(retain)(THCState *state, THCTensor *self); diff --git a/aten/src/THC/generic/THCTensorScatterGather.cu b/aten/src/THC/generic/THCTensorScatterGather.cu index 279005b6650d3..f8f75ef7edfb9 100644 --- a/aten/src/THC/generic/THCTensorScatterGather.cu +++ b/aten/src/THC/generic/THCTensorScatterGather.cu @@ -14,10 +14,8 @@ void THCTensor_(gather)(THCState* state, THCTensor *tensor, THArgCheck(THCudaLongTensor_nDimensionLegacyNoScalars(state, index) == THCTensor_(nDimensionLegacyNoScalars)(state, src), 4, "Index tensor must have same dimensions as input tensor"); - THLongStorage *indexSize = THCudaLongTensor_newSizeOf(state, index); - THArgCheck(THCTensor_(isSize)(state, tensor, indexSize), 4, + THArgCheck(tensor->sizes().equals(index->sizes()), 4, "Index tensor must have the same size as output tensor."); - THLongStorage_free(indexSize); THArgCheck(dim >= 0 && dim < THCTensor_(nDimensionLegacyNoScalars)(state, tensor), 3, "Index dimension is out of bounds"); THArgCheck(THCTensor_(nDimensionLegacyNoScalars)(state, src) == THCTensor_(nDimensionLegacyNoScalars)(state, tensor), 2, diff --git a/aten/src/THC/generic/THCTensorSort.cu b/aten/src/THC/generic/THCTensorSort.cu index 2943a5cf441d7..e1bb29bd7dbbf 100644 --- a/aten/src/THC/generic/THCTensorSort.cu +++ b/aten/src/THC/generic/THCTensorSort.cu @@ -9,10 +9,8 @@ THC_API void THCTensor_(sortKeyValueInplace)(THCState* state, THCTensor* key, THCudaLongTensor* value, int dim, bool dir) { - THLongStorage *valueSize = THCudaLongTensor_newSizeOf(state, value); - THArgCheck(THCTensor_(isSize)(state, key, valueSize), 2, + THArgCheck(key->sizes().equals(value->sizes()), 2, "Key tensor must have same size as value tensor"); - THLongStorage_free(valueSize); int dims = THCudaLongTensor_nDimensionLegacyNoScalars(state, value); THArgCheck(dims <= MAX_CUTORCH_DIMS, 3, CUTORCH_DIM_WARNING); dims = THCTensor_(nDimensionLegacyNoScalars)(state, key); diff --git a/aten/src/THCUNN/common.h b/aten/src/THCUNN/common.h index e2a99640ba69b..e8a98079b85b0 100644 --- a/aten/src/THCUNN/common.h +++ b/aten/src/THCUNN/common.h @@ -18,12 +18,10 @@ inline int GET_BLOCKS(const int N) } #define THCUNN_resizeAs_indices(STATE, I1, I2) \ - THLongStorage *size2 = THCTensor_(newSizeOf)(STATE, I2); \ - if (!THCIndexTensor_(isSize)(STATE, I1, size2)) \ + if (!I1->sizes().equals(I2->sizes())) \ { \ - THCudaLongTensor_resize(STATE, I1, size2, NULL); \ - } \ - THLongStorage_free(size2); + THCudaLongTensor_resizeAs(STATE, I1, I2); \ + } #define THCUNN_check_shape(STATE, I1, I2) \ if (I1 != NULL && I2 != NULL && !THCTensor_(isSameSizeAs)(STATE, I1, I2)) \ @@ -36,15 +34,13 @@ inline int GET_BLOCKS(const int N) #define THCUNN_check_shape_indices(STATE, I1, I2) \ - THLongStorage *size2 = THCTensor_(newSizeOf)(STATE, I2); \ - if (!THCIndexTensor_(isSize)(STATE, I1, size2)) \ + if (!I1->sizes().equals(I2->sizes())) \ { \ THCDescBuff s1 = THCIndexTensor_(sizeDesc)(STATE, I1); \ THCDescBuff s2 = THCTensor_(sizeDesc)(STATE, I2); \ THError(#I1 " and " #I2 " shapes do not match: " \ #I1 " %s, " #I2 " %s", s1.str, s2.str); \ - } \ - THLongStorage_free(size2); + } #define THCUNN_check_nElement(STATE, I1, I2) \ if (I1 != NULL && I2 != NULL ) { \ diff --git a/aten/src/THNN/THNN.h b/aten/src/THNN/THNN.h index e216e6204196b..bf0f156579b78 100644 --- a/aten/src/THNN/THNN.h +++ b/aten/src/THNN/THNN.h @@ -19,14 +19,6 @@ typedef int64_t THIndex_t; typedef int32_t THInteger_t; typedef void THNNState; -#define THNN_resizeAs_indices(I1, I2) \ - THLongStorage *size2 = THIndexTensor_(newSizeOf)(I2); \ - if (!THTensor_(isSize)(I1, size2)) \ - { \ - THTensor_(resize)(I1, size2, NULL); \ - } \ - THLongStorage_free(size2); - #include "generic/THNN.h" #include diff --git a/aten/src/THNN/generic/MultiLabelMarginCriterion.c b/aten/src/THNN/generic/MultiLabelMarginCriterion.c index a18252b06914d..cd0ecbb1e9df0 100644 --- a/aten/src/THNN/generic/MultiLabelMarginCriterion.c +++ b/aten/src/THNN/generic/MultiLabelMarginCriterion.c @@ -43,7 +43,9 @@ void THNN_(MultiLabelMarginCriterion_updateOutput)( input_data = THTensor_(data)(input); target_data = THIndexTensor_(data)(target); - THNN_resizeAs_indices(isTarget, target); + if (!isTarget->sizes().equals(target->sizes())) { + THTensor_(resizeNd)(isTarget, target->dim(), THTensor_getSizePtr(target), nullptr); + } THTensor_(zero)(isTarget); isTarget_data = THTensor_(data)(isTarget); diff --git a/aten/src/THNN/init.cpp b/aten/src/THNN/init.cpp index c77cd76d54ec8..e886272298a48 100644 --- a/aten/src/THNN/init.cpp +++ b/aten/src/THNN/init.cpp @@ -17,16 +17,12 @@ } #define THNN_CHECK_SHAPE_INDICES(I1, I2) \ - THLongStorage *size2 = THLongTensor_newSizeOf(I2); \ - if (I1 != NULL && I2 != NULL && !THTensor_(isSize)(I1, size2)) \ + if (I1 != NULL && I2 != NULL && !I1->sizes().equals(I2->sizes())) \ { \ THDescBuff s1 = THTensor_(sizeDesc)(I1); \ THDescBuff s2 = THLongTensor_sizeDesc(I2); \ - THLongStorage_free(size2); \ THError(#I1 " and " #I2 " shapes do not match: " \ #I1 " %s, " #I2 " %s", s1.str, s2.str); \ - } else { \ - THLongStorage_free(size2); \ } #define THNN_CHECK_NELEMENT(I1, I2) \ From 170d29769b6f31f191a99461f7e7e51f005b8a8d Mon Sep 17 00:00:00 2001 From: Elias Ellison Date: Thu, 2 Aug 2018 10:59:36 -0700 Subject: [PATCH 05/16] Strings lexing, parsing, implementation in print (#9324) Summary: This PR adds strings to the ast and implements them for print statements. Strings are lifted as attributes to the print node. They must be arguments to print itself, not as an argument for an object that is passed to print. If they are encountered elsewhere a NYI exception will be thrown. Pull Request resolved: https://github.com/pytorch/pytorch/pull/9324 Reviewed By: jramseyer Differential Revision: D8807128 Pulled By: eellison fbshipit-source-id: 984401ff458ed18d473c6d1bd86750e56c77d078 --- setup.py | 1 + test/expect/TestScript.test_string_cu.expect | 7 ++ ...TestScript.test_string_print-stdout.expect | 2 + test/test_jit.py | 36 +++++++ torch/CMakeLists.txt | 1 + torch/csrc/jit/constants.cpp | 9 ++ torch/csrc/jit/interned_strings.h | 4 +- torch/csrc/jit/ir.cpp | 16 ++- torch/csrc/jit/ir.h | 1 - torch/csrc/jit/ivalue.cpp | 20 ++++ torch/csrc/jit/ivalue.h | 99 ++++++++++++++++++- torch/csrc/jit/pybind_utils.h | 6 +- torch/csrc/jit/register_prim_ops.cpp | 20 +--- torch/csrc/jit/script/compiler.cpp | 23 ++++- torch/csrc/jit/script/lexer.h | 43 ++++++++ torch/csrc/jit/script/parser.h | 56 +++++++++++ torch/csrc/jit/script/python_tree_views.cpp | 4 + torch/csrc/jit/script/tree_views.h | 13 +++ torch/csrc/jit/type.cpp | 6 ++ torch/csrc/jit/type.h | 26 +++++ torch/jit/frontend.py | 6 ++ 21 files changed, 371 insertions(+), 28 deletions(-) create mode 100644 test/expect/TestScript.test_string_cu.expect create mode 100644 test/expect/TestScript.test_string_print-stdout.expect create mode 100644 torch/csrc/jit/ivalue.cpp diff --git a/setup.py b/setup.py index 2e2ef60fb4131..9691285fde67a 100644 --- a/setup.py +++ b/setup.py @@ -762,6 +762,7 @@ def run(self): "torch/csrc/finalizer.cpp", "torch/csrc/jit/batched/BatchTensor.cpp", "torch/csrc/jit/init.cpp", + "torch/csrc/jit/ivalue.cpp", "torch/csrc/jit/passes/onnx.cpp", "torch/csrc/jit/passes/onnx/fixup_onnx_loop.cpp", "torch/csrc/jit/passes/onnx/peephole.cpp", diff --git a/test/expect/TestScript.test_string_cu.expect b/test/expect/TestScript.test_string_cu.expect new file mode 100644 index 0000000000000..602ed319a1327 --- /dev/null +++ b/test/expect/TestScript.test_string_cu.expect @@ -0,0 +1,7 @@ +graph(%a : Dynamic) { + %1 : string = prim::Constant[string=a\n\tb\n]() + %2 : int = prim::Constant[value=2]() + %3 : string = prim::Constant[string=aa]() + = prim::Print(%a, %1, %2, %3) + return (%a); +} diff --git a/test/expect/TestScript.test_string_print-stdout.expect b/test/expect/TestScript.test_string_print-stdout.expect new file mode 100644 index 0000000000000..19f670510f10d --- /dev/null +++ b/test/expect/TestScript.test_string_print-stdout.expect @@ -0,0 +1,2 @@ +1 +[ Variable[CPULongType]{} ] abcd 2 1.5 diff --git a/test/test_jit.py b/test/test_jit.py index afcfb85d723c7..c05ed16b5670d 100644 --- a/test/test_jit.py +++ b/test/test_jit.py @@ -1846,6 +1846,34 @@ def foo(a): a = Variable(torch.rand(1)) self.assertEqual(a, cu.foo(a)) + # because the compilation unit ingests python strings + # to use an escape sequence escape the backslash (\\n = \n) + def test_string_cu(self): + cu = torch.jit.CompilationUnit(''' + def foo(a): + print(a, """a\\n\tb\\n""", 2, "a\ +a") + return a + ''') + self.assertExpected(str(cu.foo.graph)) + + def test_string_new_line(self): + with self.assertRaisesRegex(RuntimeError, "expected a valid token*"): + torch.jit.CompilationUnit(''' + def test_while(a): + print(" + a") + return a + ''') + + def test_string_single_escape(self): + with self.assertRaisesRegex(RuntimeError, "expected a valid token*"): + torch.jit.CompilationUnit(''' + def test_while(a): + print("\\") + return a + ''') + def test_script_annotation(self): @torch.jit.script def foo(a): @@ -2134,6 +2162,14 @@ def fn(x, y, z): def _make_scalar_vars(self, arr, dtype): return [torch.tensor(val, dtype=dtype) for val in arr] + def test_string_print(self): + def func(a): + print(a, "a" 'b' '''c''' """d""", 2, 1.5) + return a + + inputs = self._make_scalar_vars([1], torch.int64) + self.checkScript(func, inputs, capture_output=True) + def test_while(self): def func(a, b, max): while a < max: diff --git a/torch/CMakeLists.txt b/torch/CMakeLists.txt index 057bf6efeac3d..d64073dba721c 100644 --- a/torch/CMakeLists.txt +++ b/torch/CMakeLists.txt @@ -135,6 +135,7 @@ set(TORCH_SRCS ${TORCH_SRC_DIR}/csrc/jit/interpreter.cpp ${TORCH_SRC_DIR}/csrc/jit/constants.cpp ${TORCH_SRC_DIR}/csrc/jit/ir.cpp + ${TORCH_SRC_DIR}/csrc/jit/ivalue.cpp ${TORCH_SRC_DIR}/csrc/jit/operator.cpp ${TORCH_SRC_DIR}/csrc/jit/operator.cpp ${TORCH_SRC_DIR}/csrc/jit/passes/batch_mm.cpp diff --git a/torch/csrc/jit/constants.cpp b/torch/csrc/jit/constants.cpp index 47e593bbb125e..698153aad27fa 100644 --- a/torch/csrc/jit/constants.cpp +++ b/torch/csrc/jit/constants.cpp @@ -29,6 +29,9 @@ Value* insertConstant( return autograd::Variable(t).data(); })); n->output()->setType(ListType::ofTensors()); + } else if(val.isString()) { + n->s_(attr::string, val.toString()->string()); + n->output()->setType(StringType::get()); } else { throw std::runtime_error("Unsupported value kind: " + val.tagKind()); } @@ -79,6 +82,12 @@ RegisterOperators reg({ push(stack, ts); return 0; }; + } else if (type == StringType::get()) { + auto s = node->s(attr::string); + return [s](Stack& stack) { + push(stack, s); + return 0; + }; } else { std::stringstream ss; ss << "constant literal not supported for: " << type->str(); diff --git a/torch/csrc/jit/interned_strings.h b/torch/csrc/jit/interned_strings.h index c567793552d73..fd6208147dffc 100644 --- a/torch/csrc/jit/interned_strings.h +++ b/torch/csrc/jit/interned_strings.h @@ -90,7 +90,9 @@ _(attr, sizes) \ _(attr, starts) \ _(attr, transA) \ _(attr, transB) \ -_(attr, name) +_(attr, name) \ +_(attr, string) + // 'prim' symbols are synthetic operators that occur only in the IR // and don't have corresponding implementations in ATen. diff --git a/torch/csrc/jit/ir.cpp b/torch/csrc/jit/ir.cpp index ede14249c46dc..e273084be642d 100644 --- a/torch/csrc/jit/ir.cpp +++ b/torch/csrc/jit/ir.cpp @@ -82,6 +82,20 @@ void printPrimList(std::ostream & out, const std::vector & items) { } out << "]"; } + +std::string escapeString(std::string s) { + std::vector search = {'\n', '\t', '\v'}; + std::vector replace = {"\\n", "\\t", "\\v"}; + for (size_t i = 0; i < search.size(); i++) { + size_t pos = s.find(search[i]); + while(pos != std::string::npos) { + s.replace(pos, 1, replace[i]); + pos = s.find(search[i], pos + 1); + } + } + return s; +} + void printAttributes(std::ostream & out, const Node * n, bool ignore_subgraph=false) { out << "["; auto names = n->attributeNames(); @@ -110,7 +124,7 @@ void printAttributes(std::ostream & out, const Node * n, bool ignore_subgraph=fa printPrimList(out,n->is(name)); break; case AttributeKind::s: - out << n->s(name); + out << escapeString(n->s(name)); break; case AttributeKind::ss: printPrimList(out,n->ss(name)); diff --git a/torch/csrc/jit/ir.h b/torch/csrc/jit/ir.h index b2caa642b6fe2..959228e154779 100644 --- a/torch/csrc/jit/ir.h +++ b/torch/csrc/jit/ir.h @@ -987,7 +987,6 @@ friend struct Block; Node * createUndefined() { return create(prim::Undefined); } - Node * createFusionGroup(int device) { auto n = create(prim::FusionGroup, 0); n->g_(attr::Subgraph,std::make_shared(scope_root_)); diff --git a/torch/csrc/jit/ivalue.cpp b/torch/csrc/jit/ivalue.cpp new file mode 100644 index 0000000000000..57460bf1aa469 --- /dev/null +++ b/torch/csrc/jit/ivalue.cpp @@ -0,0 +1,20 @@ +#include "torch/csrc/jit/assertions.h" +#include "torch/csrc/jit/ivalue.h" +#include + +#define TORCH_FORALL_TAGS(_) \ + _(None) _(Tensor) _(Double) _(Int) _(Tuple) _(IntList) _(DoubleList) _(String) _(TensorList) + +namespace torch { namespace jit { +std::ostream& operator<<(std::ostream & out, const IValue & v) { + switch(v.tag) { + #define DEFINE_CASE(x) case IValue::Tag::x: return v.format ## x(out); + TORCH_FORALL_TAGS(DEFINE_CASE) + #undef DEFINE_CASE + } + AT_ERROR("Tag not found\n"); +} + +#undef TORCH_FORALL_TAGS + +}} diff --git a/torch/csrc/jit/ivalue.h b/torch/csrc/jit/ivalue.h index 6eef40a032306..0ad0323c1bf34 100644 --- a/torch/csrc/jit/ivalue.h +++ b/torch/csrc/jit/ivalue.h @@ -1,6 +1,7 @@ #pragma once #include "torch/csrc/jit/assertions.h" +#include "torch/csrc/WindowsTorchApiMacro.h" #include @@ -77,6 +78,28 @@ struct Shared { PointerType * pImpl; }; +// string +struct ConstantString : at::Retainable { + private: + ConstantString(const std::string & str) + : str_(str) {} + const std::string str_; + public: + static Shared create(const std::string str_) { + return Shared( + new ConstantString(str_), false); + } + const std::string & string() const { + return str_; + } + operator const std::string & () const { + return string(); + } + TORCH_API std::ostream& operator<<(std::ostream & out) const { + out << string(); + return out; + } +}; template struct ConstantList; @@ -94,7 +117,7 @@ using DoubleList = ConstantList; // retain/release calls. #define TORCH_FORALL_TAGS(_) \ - _(None) _(Tensor) _(Double) _(Int) _(Tuple) _(IntList) _(DoubleList) _(TensorList) + _(None) _(Tensor) _(Double) _(Int) _(Tuple) _(IntList) _(DoubleList) _(String) _(TensorList) struct IValue { IValue() @@ -151,6 +174,11 @@ struct IValue { JIT_ASSERT(isTensor()); return at::Tensor(as_tensor_impl, /*retain=*/true); } + TORCH_API std::ostream& formatTensor(std::ostream& out) const { + JIT_ASSERT(isTensor()); + out << toTensor(); + return out; + } // Tuple IValue(Shared v); @@ -163,6 +191,11 @@ struct IValue { JIT_ASSERT(isTuple()); return toRetainable(); } + TORCH_API std::ostream& formatTuple(std::ostream& out) const { + JIT_ASSERT(isTuple()); + out << "Tuple"; //TODO + return out; + } // Double IValue(double d) @@ -174,6 +207,11 @@ struct IValue { JIT_ASSERT(isDouble()); return as_double; } + TORCH_API std::ostream& formatDouble(std::ostream& out) const { + JIT_ASSERT(isDouble()); + out << as_double; + return out; + } // Int IValue(int64_t i) @@ -193,6 +231,12 @@ struct IValue { JIT_ASSERT(isInt()); return as_int; } + TORCH_API std::ostream& formatInt(std::ostream& out) const { + JIT_ASSERT(isInt()); + out << as_int; + return out; + } + // IntList IValue(Shared v); @@ -208,9 +252,32 @@ struct IValue { JIT_ASSERT(isIntList()); return toRetainable(); } + TORCH_API std::ostream& formatIntList(std::ostream& out) const { + JIT_ASSERT(isIntList()); + out << "Int List"; //FIXME @eellison toRetainable(); + return out; + } std::vector copyToIntList() const; + // ConstantString + IValue(Shared v); + IValue(const std::string& v); + bool isString() const { return Tag::String == tag; } + Shared toString() && { + JIT_ASSERT(isString()); + return moveToRetainable(); + } + Shared toString() const & { + JIT_ASSERT(isString()); + return toRetainable(); + } + TORCH_API std::ostream& formatString(std::ostream& out) const { + JIT_ASSERT(isString()); + out << toRetainable()->string(); + return out; + } + // DoubleList IValue(Shared v); IValue(std::vector v); @@ -223,6 +290,12 @@ struct IValue { JIT_ASSERT(isDoubleList()); return toRetainable(); } + TORCH_API std::ostream& formatDoubleList(std::ostream& out) const { + JIT_ASSERT(isDoubleList()); + out << "Double List"; //FIXME @eellison toRetainable(); + return out; + } + //TensorList IValue(Shared v); @@ -236,11 +309,20 @@ struct IValue { JIT_ASSERT(isTensorList()); return toRetainable(); } + TORCH_API std::ostream& formatTensorList(std::ostream& out) const { + JIT_ASSERT(isTensorList()); + out << "Tensor List"; //FIXME @eellison toRetainable(); + return out; + } // None bool isNone() { return Tag::None == tag; } + std::ostream& formatNone(std::ostream& out) const { + out << "None"; + return out; + } // Scalar, which gets encoded as either an Int or a Double IValue(at::Scalar s) @@ -264,7 +346,7 @@ struct IValue { } // for debugging - std::string tagKind() { + std::string tagKind() const { switch(tag) { #define DEFINE_CASE(x) case Tag::x: return #x; TORCH_FORALL_TAGS(DEFINE_CASE) @@ -287,6 +369,8 @@ struct IValue { template T to() const &; + TORCH_API friend std::ostream& operator<<(std::ostream & out, const IValue & v); + private: template Shared moveToRetainable() { @@ -339,10 +423,12 @@ DEFINE_TO(double, toDouble) DEFINE_TO(int64_t, toInt) DEFINE_TO(Shared, toDoubleList) DEFINE_TO(Shared, toIntList) +DEFINE_TO(Shared, toString) DEFINE_TO(at::Scalar, toScalar) DEFINE_TO(bool, toInt) DEFINE_TO(std::vector, copyToIntList) + #undef DEFINE_TO // non-mutable list @@ -365,6 +451,7 @@ struct ConstantList : at::Retainable { } }; + inline IValue::IValue(Shared v) : tag(Tag::Tuple), retainable(true) { as_retainable = v.detach(); @@ -377,6 +464,13 @@ inline IValue::IValue(Shared v) inline IValue::IValue(std::vector v) : IValue(IntList::create(std::move(v))) {} +inline IValue::IValue(Shared v) +: tag(Tag::String), retainable(true) { + as_retainable = v.detach(); +} +inline IValue::IValue(const std::string& v) +: IValue(ConstantString::create(v)) {} + inline IValue::IValue(Shared v) : tag(Tag::DoubleList), retainable(true) { as_retainable = v.detach(); @@ -395,4 +489,5 @@ inline std::vector IValue::copyToIntList() const { return toIntList()->elements().vec(); } + }} diff --git a/torch/csrc/jit/pybind_utils.h b/torch/csrc/jit/pybind_utils.h index 0598e651d3243..94c4d53be24cb 100644 --- a/torch/csrc/jit/pybind_utils.h +++ b/torch/csrc/jit/pybind_utils.h @@ -20,9 +20,10 @@ inline Stack createStack(const py::tuple& tuple, at::ArrayRef inputs, si return py::cast(obj); case TypeKind::NoneType: return {}; + case TypeKind::StringType: case TypeKind::ListType: case TypeKind::TupleType: - throw std::runtime_error("Lists and tuples are not supported yet"); + throw std::runtime_error("Lists tuples and strings are not supported yet"); case TypeKind::NumberType: throw std::runtime_error("Insufficient type information to convert input"); } @@ -52,9 +53,10 @@ inline py::object wrapStack(Stack&& outputs, at::ArrayRef output_vals) { return py::cast(ivalue.toInt()); case TypeKind::NoneType: return py::none(); + case TypeKind::StringType: case TypeKind::ListType: case TypeKind::TupleType: - throw std::runtime_error("Lists and tuples are not supported yet"); + throw std::runtime_error("Lists tuples and strings are not supported yet"); case TypeKind::NumberType: throw std::runtime_error("Insufficient type information to convert input"); } diff --git a/torch/csrc/jit/register_prim_ops.cpp b/torch/csrc/jit/register_prim_ops.cpp index f2b8ea18a2be2..82ba798bfc753 100644 --- a/torch/csrc/jit/register_prim_ops.cpp +++ b/torch/csrc/jit/register_prim_ops.cpp @@ -90,33 +90,17 @@ RegisterOperators reg({ return 0; }; }), - Operator( - prim::None, - [](Node* node) { - return [](Stack& stack) { - stack.push_back(IValue()); - return 0; - }; - }), Operator( prim::Print, [](Node* node) { size_t num_inputs = node->inputs().size(); return [num_inputs](Stack& stack) { bool first = true; - for (const IValue& i_ : last(stack, num_inputs)) { - auto i = i_.toTensor(); + for (const IValue& i : last(stack, num_inputs)) { if (!first) std::cout << " "; first = false; - if (auto tensor_impl = dynamic_cast(i.get())) { - std::cout << at::Tensor(tensor_impl, true); - } else if (!i.defined()) { - std::cout << ""; - } else { - auto& r = *i.get(); - std::cout << "<" << typeid(r).name() << " at " << i << ">"; - } + std::cout << i; } drop(stack, num_inputs); std::cout << std::endl; diff --git a/torch/csrc/jit/script/compiler.cpp b/torch/csrc/jit/script/compiler.cpp index 9e3c3f7ea5f74..63acb23fba044 100644 --- a/torch/csrc/jit/script/compiler.cpp +++ b/torch/csrc/jit/script/compiler.cpp @@ -46,9 +46,19 @@ struct PrintValue : public SugaredValue { auto& g = *m.graph(); if (!attributes.empty()) throw ErrorReport(loc) << "print doesn't accept any keyword arguments"; - auto values = toValues(inputs); - ensureTensors(loc, values); - g.insertNode(g.create(prim::Print, values, 0) + + //temporary hack to allow print statements to work in python 2, where + //print(a, b) is treated as a (a, b) tuple input. + + std::vector lowered_inputs = toValues(inputs); + if(lowered_inputs.size() == 1 && lowered_inputs.at(0)->node()->kind() == prim::TupleConstruct) { + auto input = lowered_inputs[0]; + for(size_t j = 0; j < input->node()->inputs().size(); ++j) { + lowered_inputs.insert(lowered_inputs.begin() + 1 + j, input->node()->inputs().at(j)); + } + lowered_inputs.erase(lowered_inputs.begin()); + } + g.insertNode(g.create(prim::Print, lowered_inputs, 0) ->setSourceLocation(std::make_shared(loc))); return std::make_shared(); } @@ -1364,6 +1374,9 @@ struct to_ir { case TK_IF_EXPR: { return emitTernaryIf(TernaryIf(tree)); } break; + case TK_STRINGLITERAL: { + return emitStringLiteral(StringLiteral(tree)); + } break; case TK_LIST_LITERAL: { auto ll = ListLiteral(tree); auto values = getValues(ll.inputs(), /*maybe_unpack=*/true, identity); @@ -1394,6 +1407,10 @@ struct to_ir { return insertConstant(*graph, c.asIntegral(), c.range()); } + Value* emitStringLiteral(const StringLiteral& c) { + return insertConstant(*graph, c.text(), c.range()); + } + // Desugars slice syntactic sugar tensor[begin:end] -> tensor.slice(begin, // end). Value* emitSlice( diff --git a/torch/csrc/jit/script/lexer.h b/torch/csrc/jit/script/lexer.h index 1694889d630d3..5543f104ea37d 100644 --- a/torch/csrc/jit/script/lexer.h +++ b/torch/csrc/jit/script/lexer.h @@ -34,6 +34,7 @@ namespace script { _(TK_EQUIVALENT, "equivalent", "<=>") \ _(TK_IDENT, "ident", "") \ _(TK_STRING, "string", "") \ + _(TK_STRINGLITERAL, "string_literal", "") \ _(TK_CONST, "const", "") \ _(TK_LIST, "list", "") \ _(TK_OPTION, "option", "") \ @@ -187,6 +188,42 @@ struct SharedParserData { *len = endptr - startptr; return *len > 0; } + + bool isCharCount(char c, const std::string& str, size_t start, int len) { + //count checks from [start, start + len) + return start + len <= str.size() && std::count(str.begin() + start, str.begin() + start + len, c) == len; + } + + // python conconcatenates all adjacent strings "a" "b" == "ab" + // strings can be enclosed with 1 or 3 single or double quotes + // if enclosed with 3 quotes newlines are valid + // as elsewhere, backslash and new line should be ignored + bool isString(const std::string& str, size_t start, size_t* len) { + char quote = str[start]; + if (quote != '\"' && quote != '\'') + return false; + int quote_len = isCharCount(quote, str, start, 3) ? 3 : 1; + + //end is now set past the opening quotation marks + size_t end = start + quote_len; + while(end < str.size() && !isCharCount(quote, str, end, quote_len)) { + if (str[end] == '\n' && quote_len != 3) { + return false; + } + //handle escaped characters. advances past escaped quotation marks, + //escaped newlines and escaped backslashes + if (str[end] == '\\') { + end++; + } + end++; + } + //set length equal to the complete string including quotations + *len = end - start + quote_len; + //if end finished without going past the last character of the string than + //there is a match + return end < str.size(); + } + bool isblank(int n) { return isspace(n) && n != '\n'; } @@ -244,6 +281,12 @@ struct SharedParserData { *kind = TK_NUMBER; return true; } + // check for string + if (isString(str, pos, len)) { + *kind = TK_STRINGLITERAL; + return true; + } + // check for either an ident or a token // ident tracks whether what we have scanned so far could be an identifier // matched indicates if we have found any match. diff --git a/torch/csrc/jit/script/parser.h b/torch/csrc/jit/script/parser.h index 0cd833dc15e48..189b72018220b 100644 --- a/torch/csrc/jit/script/parser.h +++ b/torch/csrc/jit/script/parser.h @@ -75,6 +75,9 @@ struct Parser { auto list = parseList('[', ',', ']', &Parser::parseExp); prefix = ListLiteral::create(list.range(), List(list)); } break; + case TK_STRINGLITERAL: { + prefix = parseStringLiteral(); + } break; default: { Ident name = parseIdent(); prefix = Var::create(name.range(), name); @@ -180,14 +183,67 @@ struct Parser { L.expect(end); return List::create(r, elements); } + Const parseConst() { auto range = L.cur().range; auto t = L.expect(TK_NUMBER); return Const::create(t.range, t.text()); } + + bool isCharCount(char c, const std::string& str, size_t start, int len) { + //count checks from [start, start + len) + return start + len <= str.size() && std::count(str.begin() + start, str.begin() + start + len, c) == len; + } + + std::string parseString(const SourceRange& range, const std::string &str) { + int quote_len = isCharCount(str[0], str, 0, 3) ? 3 : 1; + auto ret_str = str.substr(quote_len, str.size() - quote_len * 2); + size_t pos = ret_str.find('\\'); + while(pos != std::string::npos) { + //invariant: pos has to escape a character because it is a valid string + char c = ret_str[pos + 1]; + switch (ret_str[pos + 1]) { + case '\\': + case '\'': + case '\"': + case '\n': + break; + case 'a': + c = '\a'; + break; + case 'b': + c = '\b'; + break; + case 'f': + c = '\f'; + break; + case 'n': + c = '\n'; + break; + case 'v': + c = '\v'; + break; + default: + throw ErrorReport(range) << " octal and hex escaped sequences are not supported"; + } + ret_str.replace(pos, /* num to erase */ 2, /* num copies */ 1, c); + pos = ret_str.find('\\', pos + 1); + } + return ret_str; + } + + StringLiteral parseStringLiteral() { + auto range = L.cur().range; + std::stringstream ss; + while(L.cur().kind == TK_STRINGLITERAL) + ss << parseString(L.cur().range, L.next().text()); + return StringLiteral::create(range, ss.str()); + } + Expr parseAttributeValue() { return parseExp(); } + void parseOperatorArguments(TreeList& inputs, TreeList& attributes) { L.expect('('); if (L.cur().kind != ')') { diff --git a/torch/csrc/jit/script/python_tree_views.cpp b/torch/csrc/jit/script/python_tree_views.cpp index 7ece5e055a33d..b49673de25fc0 100644 --- a/torch/csrc/jit/script/python_tree_views.cpp +++ b/torch/csrc/jit/script/python_tree_views.cpp @@ -174,6 +174,10 @@ void initTreeViewBindings(PyObject *module) { .def(py::init([](const SourceRange& range, std::string value) { return Const::create(range, value); })); + py::class_(m, "StringLiteral") + .def(py::init([](const SourceRange& range, std::string value) { + return StringLiteral::create(range, value); + })); py::class_(m, "Apply") .def(py::init([](const Expr& expr, std::vector args, std::vector kwargs) { auto r = expr.range(); diff --git a/torch/csrc/jit/script/tree_views.h b/torch/csrc/jit/script/tree_views.h index 10ac01799c060..2ab648adf2e8d 100644 --- a/torch/csrc/jit/script/tree_views.h +++ b/torch/csrc/jit/script/tree_views.h @@ -246,6 +246,7 @@ struct Expr : public TreeView { case '/': case TK_NOT: case TK_CONST: + case TK_STRINGLITERAL: case TK_TRUE: case TK_FALSE: case TK_NONE: @@ -564,6 +565,18 @@ struct Const : public Expr { } }; +struct StringLiteral : public Expr { + explicit StringLiteral(const TreeRef& tree) : Expr(tree) { + tree_->matchNumSubtrees(TK_STRINGLITERAL, 1); + } + const std::string& text() const { + return subtree(0)->stringValue(); + } + static StringLiteral create(const SourceRange& range, const std::string& value) { + return StringLiteral(Compound::create(TK_STRINGLITERAL, range, {String::create(value)})); + } +}; + struct Apply : public Expr { explicit Apply(const TreeRef& tree) : Expr(tree) { tree_->match(TK_APPLY); diff --git a/torch/csrc/jit/type.cpp b/torch/csrc/jit/type.cpp index ddb4dfad0154a..7f246ce518a2f 100644 --- a/torch/csrc/jit/type.cpp +++ b/torch/csrc/jit/type.cpp @@ -40,6 +40,8 @@ std::ostream& operator<<(std::ostream & out, const Type & t) { out << *prim << "[]"; } else if(t.kind() == TypeKind::NoneType) { out << "None"; + } else if(t.kind() == TypeKind::StringType) { + out << "string"; } else { AT_ERROR("unknown type kind"); } @@ -66,6 +68,10 @@ NoneTypePtr NoneType::get() { static auto value = NoneType::create(); return value; } +StringTypePtr StringType::get() { + static auto value = StringType::create(); + return value; +} ListTypePtr ListType::ofTensors() { static auto value = ListType::create(DynamicType::get()); return value; diff --git a/torch/csrc/jit/type.h b/torch/csrc/jit/type.h index 713718e40681c..4febad6693845 100644 --- a/torch/csrc/jit/type.h +++ b/torch/csrc/jit/type.h @@ -21,6 +21,7 @@ _(NumberType) \ _(FloatType) \ _(IntType) \ _(NoneType) \ +_(StringType) \ enum class TypeKind { #define DEFINE_TYPE(T) T, @@ -382,6 +383,31 @@ struct TORCH_API IntType : public Type { : Type(TypeKind::IntType) {} }; +struct StringType; +using StringTypePtr = std::shared_ptr; +// This node represents a Python string value +struct TORCH_API StringType : public Type { + template + static StringTypePtr create( T&& ... all ) { + return StringTypePtr(new StringType( std::forward(all)... )); + } + bool operator==(const Type& rhs) const override { + return rhs.kind() == kind(); + } + std::string str() const override { + return "string"; + } + bool isSubtypeOf(const TypePtr rhs) const override { + return *this == *rhs; + } + static const TypeKind Kind = TypeKind::StringType; + // global singleton + static StringTypePtr get(); +private: + StringType() + : Type(TypeKind::StringType) {} +}; + struct NoneType; using NoneTypePtr = std::shared_ptr; // This node represents a Python int number value diff --git a/torch/jit/frontend.py b/torch/jit/frontend.py index bc979d1514112..9cae1878139ef 100644 --- a/torch/jit/frontend.py +++ b/torch/jit/frontend.py @@ -444,6 +444,12 @@ def build_Num(ctx, expr): r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + len(value)) return Const(r, value) + @staticmethod + def build_Str(ctx, expr): + value = str(expr.s) + r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1) + return StringLiteral(r, value) + @staticmethod def build_Starred(ctx, expr): r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1) From a243e517fa93a0d9adee2acf77b7b61b4b337794 Mon Sep 17 00:00:00 2001 From: Gregory Chanan Date: Thu, 2 Aug 2018 11:18:01 -0700 Subject: [PATCH 06/16] Guard sizes/strides in TH/THC for scalars. Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/10145 Differential Revision: D9125791 Pulled By: gchanan fbshipit-source-id: d0b8c88c49d7af85971a4531a63fd85a97bfbec7 --- aten/src/TH/THTensor.hpp | 71 +++++++++--------- aten/src/TH/THTensorApply.h | 10 +-- aten/src/TH/THTensorDimApply.h | 56 +++++++------- aten/src/TH/generic/THTensor.cpp | 12 +-- aten/src/TH/generic/THTensorApply.hpp | 6 +- aten/src/TH/generic/THTensorEvenMoreMath.cpp | 17 +++-- aten/src/TH/generic/THTensorFastGetSet.hpp | 4 + aten/src/TH/generic/THTensorLapack.cpp | 10 +-- aten/src/TH/generic/THTensorMath.cpp | 14 ++-- aten/src/TH/generic/THTensorMoreMath.cpp | 74 +++++++++---------- aten/src/THC/THCDeviceTensorUtils.cuh | 4 +- aten/src/THC/THCReduce.cuh | 4 +- aten/src/THC/THCTensor.cpp | 15 +++- aten/src/THC/THCTensor.hpp | 2 + aten/src/THC/THCTensorMathReduce.cuh | 20 ++--- aten/src/THC/THCTensorSort.cu | 2 +- aten/src/THC/THCTensorTypeUtils.cuh | 4 +- aten/src/THC/generic/THCTensor.cpp | 4 +- aten/src/THC/generic/THCTensorIndex.cu | 16 ++-- aten/src/THC/generic/THCTensorMath.cu | 6 +- aten/src/THC/generic/THCTensorMathBlas.cu | 24 +++--- aten/src/THC/generic/THCTensorMathMagma.cu | 2 +- .../src/THC/generic/THCTensorMathPointwise.cu | 4 +- aten/src/THC/generic/THCTensorMathReduce.cu | 4 +- aten/src/THC/generic/THCTensorMathScan.cu | 10 +-- aten/src/THC/generic/THCTensorMode.cu | 12 +-- aten/src/THC/generic/THCTensorRandom.cu | 6 +- .../src/THC/generic/THCTensorScatterGather.cu | 18 ++--- aten/src/THC/generic/THCTensorSort.cu | 8 +- aten/src/THC/generic/THCTensorTopK.cu | 2 +- aten/src/THNN/generic/AbsCriterion.c | 2 +- aten/src/THNN/generic/BCECriterion.c | 2 +- aten/src/THNN/generic/ClassNLLCriterion.c | 10 +-- aten/src/THNN/generic/DistKLDivCriterion.c | 2 +- .../THNN/generic/MultiLabelMarginCriterion.c | 4 +- aten/src/THNN/generic/MultiMarginCriterion.c | 4 +- aten/src/THNN/generic/SmoothL1Criterion.c | 2 +- aten/src/THNN/generic/SoftMarginCriterion.c | 2 +- .../THNN/generic/SpatialClassNLLCriterion.c | 6 +- 39 files changed, 250 insertions(+), 225 deletions(-) diff --git a/aten/src/TH/THTensor.hpp b/aten/src/TH/THTensor.hpp index 56204a00e9c3e..9b1584f9c342e 100644 --- a/aten/src/TH/THTensor.hpp +++ b/aten/src/TH/THTensor.hpp @@ -127,41 +127,6 @@ inline THStorage* THTensor_getStoragePtr(const THTensor* tensor) { return tensor->storage_; } -#include "generic/THTensorFastGetSet.hpp" -#include "THGenerateAllTypes.h" - -inline void THTensor_resizeDim(THTensor* tensor, int64_t ndim) { - // NB: This is *truly* a resize; calling code (e.g., squeeze) - // assumes that old values are preserved - tensor->is_zero_dim_ = bool(ndim == 0); - tensor->sizes_.resize(ndim); - tensor->strides_.resize(ndim); -} - -inline void THTensor_setSizesAndStrides(THTensor* tensor, std::vector&& new_size, std::vector&& new_stride) { - tensor->sizes_ = std::move(new_size); - tensor->strides_ = std::move(new_stride); -} - -inline void THTensor_setSizeAtDim(THTensor* tensor, int dim, int64_t new_size) { - tensor->sizes_[dim] = new_size; -} - -inline void THTensor_setStrideAtDim(THTensor* tensor, int dim, int64_t new_stride) { - tensor->strides_[dim] = new_stride; -} - -inline void THTensor_setStorageOffset(THTensor* tensor, ptrdiff_t storage_offset) { - tensor->storage_offset_ = storage_offset; -} - -// NB: Steals ownership of storage -inline void THTensor_stealAndSetStoragePtr(THTensor* tensor, THStorage* storage) { - // Caffe2 might have tensors whose storages are null, but we - // don't allow it in PyTorch. - AT_ASSERT(storage); - tensor->storage_ = storage; -} inline bool THTensor_isZeroDim(const THTensor *tensor) { return tensor->is_zero_dim_; @@ -209,6 +174,42 @@ inline int64_t THTensor_sizeLegacyNoScalars(const THTensor *self, int dim) return THTensor_isZeroDim(self) ? 1 : self->size(dim); } +#include "generic/THTensorFastGetSet.hpp" +#include "THGenerateAllTypes.h" + +inline void THTensor_resizeDim(THTensor* tensor, int64_t ndim) { + // NB: This is *truly* a resize; calling code (e.g., squeeze) + // assumes that old values are preserved + tensor->is_zero_dim_ = bool(ndim == 0); + tensor->sizes_.resize(ndim); + tensor->strides_.resize(ndim); +} + +inline void THTensor_setSizesAndStrides(THTensor* tensor, std::vector&& new_size, std::vector&& new_stride) { + tensor->sizes_ = std::move(new_size); + tensor->strides_ = std::move(new_stride); +} + +inline void THTensor_setSizeAtDim(THTensor* tensor, int dim, int64_t new_size) { + tensor->sizes_[dim] = new_size; +} + +inline void THTensor_setStrideAtDim(THTensor* tensor, int dim, int64_t new_stride) { + tensor->strides_[dim] = new_stride; +} + +inline void THTensor_setStorageOffset(THTensor* tensor, ptrdiff_t storage_offset) { + tensor->storage_offset_ = storage_offset; +} + +// NB: Steals ownership of storage +inline void THTensor_stealAndSetStoragePtr(THTensor* tensor, THStorage* storage) { + // Caffe2 might have tensors whose storages are null, but we + // don't allow it in PyTorch. + AT_ASSERT(storage); + tensor->storage_ = storage; +} + TH_API void THTensor_free(THTensor *self); TH_CPP_API at::optional> THTensor_compute_stride(at::IntList oldshape, at::IntList oldstride, at::IntList newshape); diff --git a/aten/src/TH/THTensorApply.h b/aten/src/TH/THTensorApply.h index 9b80323a238ed..144c7c67797a3 100644 --- a/aten/src/TH/THTensorApply.h +++ b/aten/src/TH/THTensorApply.h @@ -47,9 +47,9 @@ TENSOR##_size = 1; \ TENSOR##_stride = 1; \ for(TENSOR##_i = THTensor_nDimensionLegacyAll(TENSOR)-1; TENSOR##_i >= 0; TENSOR##_i--) { \ - if(TENSOR->size(TENSOR##_i) != 1) { \ - if(TENSOR->stride(TENSOR##_i) == TENSOR##_size && TENSOR##_i != DIM) \ - TENSOR##_size *= TENSOR->size(TENSOR##_i); \ + if(THTensor_sizeLegacyNoScalars(TENSOR, TENSOR##_i) != 1) { \ + if(THTensor_strideLegacyNoScalars(TENSOR, TENSOR##_i) == TENSOR##_size && TENSOR##_i != DIM) \ + TENSOR##_size *= THTensor_sizeLegacyNoScalars(TENSOR, TENSOR##_i); \ else{ \ TENSOR##_contiguous = 0; \ break; \ @@ -70,8 +70,8 @@ TENSOR##_strides = TENSOR##_counter + 2*TENSOR##_dim; \ TH_TENSOR_dim_index = TENSOR##_dim-1; \ TENSOR##_dimOffset = (DIM == THTensor_nDimensionLegacyAll(TENSOR)-1) ? &TENSOR##_i : &TENSOR##_counter[DIM]; \ - TENSOR##_sizes[TH_TENSOR_dim_index] = TENSOR->size(THTensor_nDimensionLegacyAll(TENSOR)-1); \ - TENSOR##_strides[TH_TENSOR_dim_index] = TENSOR->stride(THTensor_nDimensionLegacyAll(TENSOR)-1); \ + TENSOR##_sizes[TH_TENSOR_dim_index] = THTensor_sizeLegacyNoScalars(TENSOR, THTensor_nDimensionLegacyAll(TENSOR)-1); \ + TENSOR##_strides[TH_TENSOR_dim_index] = THTensor_strideLegacyNoScalars(TENSOR, THTensor_nDimensionLegacyAll(TENSOR)-1); \ /* TENSOR##_counter tracks where we are in the storage. The offset into the */ \ /* storage is given by storage_offset + (i * j), where i is the stride */ \ /* vector and j is tensor_counter vector. This sets the starting position for the loop. */ \ diff --git a/aten/src/TH/THTensorDimApply.h b/aten/src/TH/THTensorDimApply.h index ff05ed8194979..9a691a95febc3 100644 --- a/aten/src/TH/THTensorDimApply.h +++ b/aten/src/TH/THTensorDimApply.h @@ -61,16 +61,16 @@ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] = 0; \ \ TENSOR1##_data = THTensor_getStoragePtr(TENSOR1)->data()+(TENSOR1)->storage_offset(); \ - TENSOR1##_stride = (TENSOR1)->stride(DIMENSION); \ - TENSOR1##_size = TENSOR1->size(DIMENSION); \ + TENSOR1##_stride = THTensor_strideLegacyNoScalars((TENSOR1), DIMENSION); \ + TENSOR1##_size = THTensor_sizeLegacyNoScalars((TENSOR1), DIMENSION); \ \ TENSOR2##_data = THTensor_getStoragePtr(TENSOR2)->data()+(TENSOR2)->storage_offset(); \ - TENSOR2##_stride = (TENSOR2)->stride(DIMENSION); \ - TENSOR2##_size = TENSOR2->size(DIMENSION); \ + TENSOR2##_stride = THTensor_strideLegacyNoScalars((TENSOR2), DIMENSION); \ + TENSOR2##_size = THTensor_sizeLegacyNoScalars((TENSOR2), DIMENSION); \ \ TENSOR3##_data = THTensor_getStoragePtr(TENSOR3)->data()+(TENSOR3)->storage_offset(); \ - TENSOR3##_stride = (TENSOR3)->stride(DIMENSION); \ - TENSOR3##_size = TENSOR3->size(DIMENSION); \ + TENSOR3##_stride = THTensor_strideLegacyNoScalars((TENSOR3), DIMENSION); \ + TENSOR3##_size = THTensor_sizeLegacyNoScalars((TENSOR3), DIMENSION); \ \ while(!TH_TENSOR_DIM_APPLY_hasFinished) \ { \ @@ -92,11 +92,11 @@ } \ \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]++; \ - TENSOR1##_data += TENSOR1->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR2##_data += TENSOR2->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR3##_data += TENSOR3->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR1##_data += THTensor_strideLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i); \ + TENSOR2##_data += THTensor_strideLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i); \ + TENSOR3##_data += THTensor_strideLegacyNoScalars(TENSOR3, TH_TENSOR_DIM_APPLY_i); \ \ - if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == TENSOR1->size(TH_TENSOR_DIM_APPLY_i)) \ + if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == THTensor_sizeLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i)) \ { \ if(TH_TENSOR_DIM_APPLY_i == THTensor_nDimensionLegacyNoScalars(TENSOR1)-1) \ { \ @@ -105,9 +105,9 @@ } \ else \ { \ - TENSOR1##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR1->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR2##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR2->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR3##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR3->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR1##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i); \ + TENSOR2##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i); \ + TENSOR3##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR3, TH_TENSOR_DIM_APPLY_i); \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] = 0; \ } \ } \ @@ -155,7 +155,7 @@ { \ if(TH_TENSOR_DIM_APPLY_i == DIMENSION) \ continue; \ - if(TENSOR1->size(TH_TENSOR_DIM_APPLY_i) != TENSOR2->size(TH_TENSOR_DIM_APPLY_i)) { \ + if(THTensor_sizeLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i) != THTensor_sizeLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i)) { \ AT_ERROR("Expected ", #TENSOR1, " ", TENSOR1->sizes(), " and ", #TENSOR2, " ", TENSOR2->sizes(), " to have the same size in dimension ", DIMENSION); \ } \ } \ @@ -168,12 +168,12 @@ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] = 0; \ \ TENSOR1##_data = THTensor_getStoragePtr(TENSOR1)->data()+(TENSOR1)->storage_offset(); \ - TENSOR1##_stride = (TENSOR1)->stride(DIMENSION); \ - TENSOR1##_size = TENSOR1->size(DIMENSION); \ + TENSOR1##_stride = THTensor_strideLegacyNoScalars((TENSOR1), DIMENSION); \ + TENSOR1##_size = THTensor_sizeLegacyNoScalars(TENSOR1, DIMENSION); \ \ TENSOR2##_data = THTensor_getStoragePtr(TENSOR2)->data()+(TENSOR2)->storage_offset(); \ - TENSOR2##_stride = (TENSOR2)->stride(DIMENSION); \ - TENSOR2##_size = TENSOR2->size(DIMENSION); \ + TENSOR2##_stride = THTensor_strideLegacyNoScalars((TENSOR2), DIMENSION); \ + TENSOR2##_size = THTensor_sizeLegacyNoScalars(TENSOR2, DIMENSION); \ \ while(!TH_TENSOR_DIM_APPLY_hasFinished) \ { \ @@ -195,10 +195,10 @@ } \ \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]++; \ - TENSOR1##_data += TENSOR1->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR2##_data += TENSOR2->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR1##_data += THTensor_strideLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i); \ + TENSOR2##_data += THTensor_strideLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i); \ \ - if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == TENSOR1->size(TH_TENSOR_DIM_APPLY_i)) \ + if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == THTensor_sizeLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i)) \ { \ if(TH_TENSOR_DIM_APPLY_i == THTensor_nDimensionLegacyNoScalars(TENSOR1)-1) \ { \ @@ -207,8 +207,8 @@ } \ else \ { \ - TENSOR1##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR1->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR2##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR2->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR1##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i); \ + TENSOR2##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i); \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] = 0; \ } \ } \ @@ -270,8 +270,8 @@ THError("invalid dimension"); \ \ TENSOR##_data = THTensor_getStoragePtr(TENSOR)->data()+(TENSOR)->storage_offset(); \ - TENSOR##_stride = (TENSOR)->stride(DIMENSION); \ - TENSOR##_size = TENSOR->size(DIMENSION); \ + TENSOR##_stride = THTensor_strideLegacyNoScalars((TENSOR), DIMENSION); \ + TENSOR##_size = THTensor_sizeLegacyNoScalars(TENSOR, DIMENSION); \ /* Counter stores the indices into the Tensor at any time */ \ TH_TENSOR_DIM_APPLY_counter = (int64_t*)THAlloc(sizeof(int64_t)*(THTensor_nDimensionLegacyAll(TENSOR))); \ for(TH_TENSOR_DIM_APPLY_i = 0; TH_TENSOR_DIM_APPLY_i < THTensor_nDimensionLegacyAll(TENSOR); TH_TENSOR_DIM_APPLY_i++) \ @@ -302,9 +302,9 @@ \ /* Bump the counter at this index, update the pointer */ \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]++; \ - TENSOR##_data += TENSOR->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR##_data += THTensor_strideLegacyNoScalars(TENSOR, TH_TENSOR_DIM_APPLY_i); \ \ - if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == TENSOR->size(TH_TENSOR_DIM_APPLY_i)) \ + if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == THTensor_sizeLegacyNoScalars(TENSOR, TH_TENSOR_DIM_APPLY_i)) \ { \ /* Handled TENSOR_size(dim) iterations for DIM_APPLY_i. If this is the last dimension, exit */ \ if(TH_TENSOR_DIM_APPLY_i == THTensor_nDimensionLegacyAll(TENSOR)-1) \ @@ -315,7 +315,7 @@ else \ { \ /* Reset the counter, and the pointer to the beginning of the storage for this combination of indices */ \ - TENSOR##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR, TH_TENSOR_DIM_APPLY_i); \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] = 0; \ } \ } \ diff --git a/aten/src/TH/generic/THTensor.cpp b/aten/src/TH/generic/THTensor.cpp index 20ddd70475119..a04e30b0dbe7c 100644 --- a/aten/src/TH/generic/THTensor.cpp +++ b/aten/src/TH/generic/THTensor.cpp @@ -439,7 +439,7 @@ void THTensor_(unfold)(THTensor *self, THTensor *src, int dimension, int64_t siz src = self; THArgCheck((dimension >= 0) && (dimension < THTensor_nDimensionLegacyNoScalars(src)), 2, "out of range"); - THArgCheck(size <= src->size(dimension), 3, "out of range"); + THArgCheck(size <= THTensor_sizeLegacyNoScalars(src, dimension), 3, "out of range"); THArgCheck(step > 0, 4, "invalid step"); THTensor_(set)(self, src); @@ -563,7 +563,7 @@ int THTensor_(isTransposed)(const THTensor *self) int64_t size_max_stride = 1; int64_t z = 1; int d; - for (d = 0; d < THTensor_nDimensionLegacyAll(self); ++d) { + for (d = 0; d < self->dim(); ++d) { if (self->stride(d) == 0 && self->size(d) != 1) return 0; if (self->stride(d) > max_stride) { @@ -756,15 +756,15 @@ void THTensor_(resizeNd)(THTensor *self, int nDimension, int64_t *size, int64_t void THTensor_(set1d)(THTensor *tensor, int64_t x0, real value) { THArgCheck(THTensor_nDimensionLegacyNoScalars(tensor) == 1, 1, "tensor must have one dimension"); - THArgCheck( (x0 >= 0) && (x0 < tensor->size(0)), 2, "out of range"); - THStorage_(set)(THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*tensor->stride(0), value); + THArgCheck( (x0 >= 0) && (x0 < THTensor_sizeLegacyNoScalars(tensor, 0)), 2, "out of range"); + THStorage_(set)(THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*THTensor_strideLegacyNoScalars(tensor, 0), value); } real THTensor_(get1d)(const THTensor *tensor, int64_t x0) { THArgCheck(THTensor_nDimensionLegacyNoScalars(tensor) == 1, 1, "tensor must have one dimension"); - THArgCheck( (x0 >= 0) && (x0 < tensor->size(0)), 2, "out of range"); - return THStorage_(get)(THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*tensor->stride(0)); + THArgCheck( (x0 >= 0) && (x0 < THTensor_sizeLegacyNoScalars(tensor, 0)), 2, "out of range"); + return THStorage_(get)(THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*THTensor_strideLegacyNoScalars(tensor, 0)); } void THTensor_(set2d)(THTensor *tensor, int64_t x0, int64_t x1, real value) diff --git a/aten/src/TH/generic/THTensorApply.hpp b/aten/src/TH/generic/THTensorApply.hpp index dd31314cb41ea..47fdaaf0df537 100644 --- a/aten/src/TH/generic/THTensorApply.hpp +++ b/aten/src/TH/generic/THTensorApply.hpp @@ -121,14 +121,14 @@ int shape_check_flag = 0; \ for(TH_TENSOR_DIM_APPLY_i = 0; TH_TENSOR_DIM_APPLY_i < THTensor_nDimensionLegacyAll(TENSOR1); TH_TENSOR_DIM_APPLY_i++) \ { \ - int64_t TENSOR3##_dim_size = TENSOR3->size(TH_TENSOR_DIM_APPLY_i); \ + int64_t TENSOR3##_dim_size = THTensor_sizeLegacyNoScalars(TENSOR3, TH_TENSOR_DIM_APPLY_i); \ if (TH_TENSOR_DIM_APPLY_i != DIMENSION) { \ - if (TENSOR3##_dim_size > TENSOR1->size(TH_TENSOR_DIM_APPLY_i)) { \ + if (TENSOR3##_dim_size > THTensor_sizeLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i)) { \ shape_check_flag = 1; \ break; \ } \ } \ - if (TENSOR3##_dim_size > TENSOR2->size(TH_TENSOR_DIM_APPLY_i)) { \ + if (TENSOR3##_dim_size > THTensor_sizeLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i)) { \ shape_check_flag = 1; \ break; \ } \ diff --git a/aten/src/TH/generic/THTensorEvenMoreMath.cpp b/aten/src/TH/generic/THTensorEvenMoreMath.cpp index 03946724dcadc..a2dfc3848ad39 100644 --- a/aten/src/TH/generic/THTensorEvenMoreMath.cpp +++ b/aten/src/TH/generic/THTensorEvenMoreMath.cpp @@ -132,8 +132,8 @@ void THTensor_(nonzero)(THLongTensor *subscript, THTensor *tensor) div = 1; for (dim = tensor->dim() - 1; dim >= 0; dim--) { - *(subscript_data + dim) = (i/div) % tensor->size(dim); - div *= tensor->size(dim); + *(subscript_data + dim) = (i/div) % THTensor_sizeLegacyNoScalars(tensor, dim); + div *= THTensor_sizeLegacyNoScalars(tensor, dim); } subscript_data += tensor->dim(); @@ -170,10 +170,11 @@ void THTensor_(indexSelect)(THTensor *tensor, THTensor *src, int dim, THLongTens { tensor_data = THTensor_(data)(tensor); src_data = THTensor_(data)(src); - ptrdiff_t rowsize = src->size(0) == 0 ? 1: THTensor_(nElement)(src) / src->size(0); + auto src_size0 = THTensor_sizeLegacyNoScalars(src, 0); + ptrdiff_t rowsize = src_size0 == 0 ? 1: THTensor_(nElement)(src) / src_size0; // check that the indices are within range - int64_t max = src->size(0) - 1 + TH_INDEX_BASE; + int64_t max = src_size0 - 1 + TH_INDEX_BASE; for (i=0; i max) { THLongTensor_free(index); @@ -422,7 +423,7 @@ void THTensor_(gather)(THTensor *tensor, THTensor *src, int dim, THLongTensor *i THArgCheck(THTensor_(nDimensionLegacyNoScalars)(src) == THTensor_(nDimensionLegacyNoScalars)(tensor), 2, "Input tensor must have same dimensions as output tensor"); - elems_per_row = THLongTensor_size(index, dim); + elems_per_row = THTensor_sizeLegacyNoScalars(index, dim); TH_TENSOR_DIM_APPLY3(real, tensor, real, src, int64_t, index, dim, TH_TENSOR_DIM_APPLY3_SIZE_EQ_EXCEPT_DIM, @@ -448,7 +449,7 @@ void THTensor_(scatter)(THTensor *tensor, int dim, THLongTensor *index, THTensor THArgCheck(THTensor_(nDimensionLegacyNoScalars)(src) == THTensor_(nDimensionLegacyNoScalars)(tensor), 4, "Input tensor must have same dimensions as output tensor"); - elems_per_row = THLongTensor_size(index, dim); + elems_per_row = THTensor_sizeLegacyNoScalars(index, dim); TH_TENSOR_DIM_APPLY3(real, tensor, real, src, int64_t, index, dim, TH_TENSOR_DIM_APPLY3_SIZE_SCATTER, @@ -474,7 +475,7 @@ void THTensor_(scatterAdd)(THTensor *tensor, int dim, THLongTensor *index, THTen THArgCheck(THTensor_(nDimensionLegacyNoScalars)(src) == THTensor_(nDimensionLegacyNoScalars)(tensor), 4, "Input tensor must have same dimensions as output tensor"); - elems_per_row = THLongTensor_size(index, dim); + elems_per_row = THTensor_sizeLegacyNoScalars(index, dim); TH_TENSOR_DIM_APPLY3(real, tensor, real, src, int64_t, index, dim, TH_TENSOR_DIM_APPLY3_SIZE_SCATTER, @@ -498,7 +499,7 @@ void THTensor_(scatterFill)(THTensor *tensor, int dim, THLongTensor *index, real THArgCheck(THLongTensor_nDimensionLegacyAll(index) == THTensor_(nDimensionLegacyAll)(tensor), 3, "Index tensor must have same dimensions as output tensor"); - elems_per_row = THLongTensor_size(index, dim); + elems_per_row = THTensor_sizeLegacyNoScalars(index, dim); TH_TENSOR_DIM_APPLY2(real, tensor, int64_t, index, dim, for (i = 0; i < elems_per_row; ++i) diff --git a/aten/src/TH/generic/THTensorFastGetSet.hpp b/aten/src/TH/generic/THTensorFastGetSet.hpp index 5ad3e13b237b8..b897aa5778a54 100644 --- a/aten/src/TH/generic/THTensorFastGetSet.hpp +++ b/aten/src/TH/generic/THTensorFastGetSet.hpp @@ -2,6 +2,10 @@ #define TH_GENERIC_FILE "generic/THTensorFastGetSet.hpp" #else +static inline real THTensor_(fastGetLegacy1dNoScalars)(THTensor *self, int64_t x0) { + return (THStorage_(data)(THTensor_getStoragePtr(self))+self->storage_offset())[(x0)*THTensor_strideLegacyNoScalars(self, 0)]; +} + static inline real THTensor_(fastGet1d)(THTensor *self, int64_t x0) { return (THStorage_(data)(THTensor_getStoragePtr(self))+self->storage_offset())[(x0)*self->stride(0)]; } diff --git a/aten/src/TH/generic/THTensorLapack.cpp b/aten/src/TH/generic/THTensorLapack.cpp index 73af3f8407080..e3855faff3a8c 100644 --- a/aten/src/TH/generic/THTensorLapack.cpp +++ b/aten/src/TH/generic/THTensorLapack.cpp @@ -373,7 +373,7 @@ void THTensor_(syev)(THTensor *re_, THTensor *rv_, THTensor *a, const char *jobz rv__ = THTensor_(cloneColumnMajor)(rv_, a); - n = rv__->size(0); + n = THTensor_sizeLegacyNoScalars(rv__, 0); lda = n; THTensor_(resize1d)(re_,n); @@ -688,7 +688,7 @@ void THTensor_(potri)(THTensor *ra_, THTensor *a, const char *uplo) ra__ = THTensor_(cloneColumnMajor)(ra_, a); - n = ra__->size(0); + n = THTensor_sizeLegacyNoScalars(ra__, 0); lda = n; /* Run inverse */ @@ -866,8 +866,8 @@ void THTensor_(orgqr)(THTensor *ra_, THTensor *a, THTensor *tau) THTensor *ra__ = NULL; ra__ = THTensor_(cloneColumnMajor)(ra_, a); - int m = ra__->size(0); - int k = tau->size(0); + int m = THTensor_sizeLegacyNoScalars(ra__, 0); + int k = THTensor_sizeLegacyNoScalars(tau, 0); int lda = m; /* Dry-run to query the suggested size of the workspace. */ @@ -921,7 +921,7 @@ void THTensor_(ormqr)(THTensor *ra_, THTensor *a, THTensor *tau, THTensor *c, co int m = c->size(0); int n = c->size(1); - int k = tau->size(0); + int k = THTensor_sizeLegacyNoScalars(tau, 0); int lda; if (*side == 'L') { diff --git a/aten/src/TH/generic/THTensorMath.cpp b/aten/src/TH/generic/THTensorMath.cpp index 24d9a7e8c4ea0..58c828120587b 100644 --- a/aten/src/TH/generic/THTensorMath.cpp +++ b/aten/src/TH/generic/THTensorMath.cpp @@ -815,10 +815,10 @@ void THTensor_(addmv)(THTensor *r_, real beta, THTensor *t, real alpha, THTensor THError("size mismatch, %s, %s", bm.str, bv.str); } - if(t->dim() != 1) + if(THTensor_nDimensionLegacyNoScalars(t) != 1) THError("vector expected, got t: %dD", t->dim()); - if(t->size(0) != mat->size(0)) { + if(THTensor_sizeLegacyNoScalars(t, 0) != mat->size(0)) { THDescBuff bt = THTensor_(sizeDesc)(t); THDescBuff bm = THTensor_(sizeDesc)(mat); THError("size mismatch, t: %s, mat: %s", bt.str, bm.str); @@ -830,6 +830,8 @@ void THTensor_(addmv)(THTensor *r_, real beta, THTensor *t, real alpha, THTensor THTensor_(copy)(r_, t); } + auto r_stride = THTensor_strideLegacyNoScalars(r_, 0); + // n == 1 || lda >= max(1, m) #define LDA_COND(M, N, LDA) ((N) == 1 || (LDA) >= THMax(1, (M))) @@ -838,14 +840,14 @@ void THTensor_(addmv)(THTensor *r_, real beta, THTensor *t, real alpha, THTensor THBlas_(gemv)('n', mat->size(0), mat->size(1), alpha, THTensor_(data)(mat), mat->stride(1), THTensor_(data)(vec), THTensor_strideLegacyNoScalars(vec, 0), - beta, THTensor_(data)(r_), r_->stride(0)); + beta, THTensor_(data)(r_), r_stride); } else if(mat->stride(1) == 1 && LDA_COND(mat->size(1), mat->size(0), mat->stride(0))) { THBlas_(gemv)('t', mat->size(1), mat->size(0), alpha, THTensor_(data)(mat), mat->stride(0), THTensor_(data)(vec), THTensor_strideLegacyNoScalars(vec, 0), - beta, THTensor_(data)(r_), r_->stride(0)); + beta, THTensor_(data)(r_), r_stride); } else { @@ -854,7 +856,7 @@ void THTensor_(addmv)(THTensor *r_, real beta, THTensor *t, real alpha, THTensor THBlas_(gemv)('t', mat->size(1), mat->size(0), alpha, THTensor_(data)(cmat), cmat->stride(0), THTensor_(data)(vec), THTensor_strideLegacyNoScalars(vec, 0), - beta, THTensor_(data)(r_), r_->stride(0)); + beta, THTensor_(data)(r_), r_stride); THTensor_(free)(cmat); } @@ -1099,7 +1101,7 @@ void THTensor_(addr)(THTensor *r_, real beta, THTensor *t, real alpha, THTensor THTensor_(data)(vec2), vec2_stride, THTensor_(data)(r_), r_->stride(1)); } - else if(r_->stride(1) == 1 && LDA_COND(vec2->size(0), vec1_size, r_->stride(0))) + else if(r_->stride(1) == 1 && LDA_COND(vec2_size, vec1_size, r_->stride(0))) { THBlas_(ger)(vec2_size, vec1_size, alpha, THTensor_(data)(vec2), vec2_stride, diff --git a/aten/src/TH/generic/THTensorMoreMath.cpp b/aten/src/TH/generic/THTensorMoreMath.cpp index fa8fb0558661e..e0dd8dc839af8 100644 --- a/aten/src/TH/generic/THTensorMoreMath.cpp +++ b/aten/src/TH/generic/THTensorMoreMath.cpp @@ -88,7 +88,7 @@ void THTensor_(max)(THTensor *values_, THLongTensor *indices_, THTensor *t, int THLongStorage_free(dim); // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { real theMax; real value; int64_t theIndex; @@ -121,7 +121,7 @@ void THTensor_(max)(THTensor *values_, THLongTensor *indices_, THTensor *t, int } THLongTensor_zero(indices_); - if(t->size(dimension) == 1) { + if(THTensor_sizeLegacyNoScalars(t, dimension) == 1) { if (!keepdim) { THTensor_(squeeze1d)(values_, values_, dimension); THLongTensor_squeeze1d(indices_, indices_, dimension); @@ -131,12 +131,12 @@ void THTensor_(max)(THTensor *values_, THLongTensor *indices_, THTensor *t, int THTensor *tempValues_ = THTensor_(newWithTensor)(values_); // tempValues_.expand_as(t) - THTensor_setSizeAtDim(tempValues_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(tempValues_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(tempValues_, dimension, 0); THLongTensor *tempIndices_ = THLongTensor_newWithTensor(indices_); // tempIndices_.expand_as(t) - THTensor_setSizeAtDim(tempIndices_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(tempIndices_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(tempIndices_, dimension, 0); TH_TENSOR_APPLY3_D(real, t, real, tempValues_, int64_t, tempIndices_, dimension, @@ -172,7 +172,7 @@ void THTensor_(min)(THTensor *values_, THLongTensor *indices_, THTensor *t, int THLongStorage_free(dim); // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { real theMax; real value; int64_t theIndex; @@ -205,7 +205,7 @@ void THTensor_(min)(THTensor *values_, THLongTensor *indices_, THTensor *t, int } THLongTensor_zero(indices_); - if(t->size(dimension) == 1) { + if(THTensor_sizeLegacyNoScalars(t, dimension) == 1) { if (!keepdim) { THTensor_(squeeze1d)(values_, values_, dimension); THLongTensor_squeeze1d(indices_, indices_, dimension); @@ -215,12 +215,12 @@ void THTensor_(min)(THTensor *values_, THLongTensor *indices_, THTensor *t, int THTensor *tempValues_ = THTensor_(newWithTensor)(values_); // tempValues_.expand_as(t) - THTensor_setSizeAtDim(tempValues_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(tempValues_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(tempValues_, dimension, 0); THLongTensor *tempIndices_ = THLongTensor_newWithTensor(indices_); // tempIndices_.expand_as(t) - THTensor_setSizeAtDim(tempIndices_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(tempIndices_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(tempIndices_, dimension, 0); TH_TENSOR_APPLY3_D(real, t, real, tempValues_, int64_t, tempIndices_, dimension, @@ -282,8 +282,8 @@ void THTensor_(sum)(THTensor *r_, THTensor *t, int dimension, int keepdim) real *t_data = tp+tBasicIndex; real *r__data = rp+iter; *r__data = 0; - for(j=0; j < t->size(dimension); ++j) { - *r__data += *(t_data + j*t->stride(dimension)); + for(j=0; j < THTensor_sizeLegacyNoScalars(t, dimension); ++j) { + *r__data += *(t_data + j*THTensor_strideLegacyNoScalars(t, dimension)); } } } else { @@ -295,7 +295,7 @@ void THTensor_(sum)(THTensor *r_, THTensor *t, int dimension, int keepdim) #endif if (serial_path) { // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { TH_TENSOR_DIM_APPLY2(real, t, real, r_, dimension, accreal sum = 0; int64_t i; @@ -306,7 +306,7 @@ void THTensor_(sum)(THTensor *r_, THTensor *t, int dimension, int keepdim) THTensor_(zero)(r_); THTensor *temp_ = THTensor_(newWithTensor)(r_); // r_.expand_as(t) - THTensor_setSizeAtDim(temp_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(temp_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(temp_, dimension, 0); TH_TENSOR_APPLY2(real, temp_, real, t, *temp__data = *temp__data + *t_data;); @@ -362,8 +362,8 @@ void THTensor_(prod)(THTensor *r_, THTensor *t, int dimension, int keepdim) real *t_data = tp+tBasicIndex; real *r__data = rp+iter; *r__data = 1; - for(j=0; j < t->size(dimension); ++j) { - *r__data *= *(t_data + j*t->stride(dimension)); + for(j=0; j < THTensor_sizeLegacyNoScalars(t, dimension); ++j) { + *r__data *= *(t_data + j*THTensor_strideLegacyNoScalars(t, dimension)); } } } else { @@ -376,7 +376,7 @@ void THTensor_(prod)(THTensor *r_, THTensor *t, int dimension, int keepdim) if(serial_path) { // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { TH_TENSOR_DIM_APPLY2(real, t, real, r_, dimension, accreal prod = 1; int64_t i; @@ -387,7 +387,7 @@ void THTensor_(prod)(THTensor *r_, THTensor *t, int dimension, int keepdim) THTensor_(fill)(r_, 1); THTensor *temp_ = THTensor_(newWithTensor)(r_); // r_.expand_as(t) - THTensor_setSizeAtDim(temp_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(temp_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(temp_, dimension, 0); TH_TENSOR_APPLY2(real, temp_, real, t, *temp__data = *temp__data * *t_data;); @@ -480,7 +480,7 @@ void THTensor_(cross)(THTensor *r_, THTensor *a, THTensor *b, int dimension) THError("inconsistent tensor dimension %dD, %dD", THTensor_(nDimensionLegacyNoScalars)(a), THTensor_(nDimensionLegacyNoScalars)(b)); - for(i = 0; i < THTensor_(nDimensionLegacyNoScalars)(a); i++) + for(i = 0; i < a->dim(); i++) { if(THTensor_(size)(a, i) != THTensor_(size)(b, i)) { THDescBuff ba = THTensor_(sizeDesc)(a); @@ -493,7 +493,7 @@ void THTensor_(cross)(THTensor *r_, THTensor *a, THTensor *b, int dimension) { for(i = 0; i < THTensor_(nDimensionLegacyNoScalars)(a); i++) { - if(THTensor_(size)(a, i) == 3) + if(THTensor_sizeLegacyNoScalars(a, i) == 3) { dimension = i; break; @@ -507,7 +507,7 @@ void THTensor_(cross)(THTensor *r_, THTensor *a, THTensor *b, int dimension) THArgCheck(dimension >= 0 && dimension < THTensor_(nDimensionLegacyNoScalars)(a), 3, "dimension %d out of range", dimension + TH_INDEX_BASE); - THArgCheck(THTensor_(size)(a, dimension) == 3, 3, "dimension %d does not have size 3", + THArgCheck(THTensor_sizeLegacyNoScalars(a, dimension) == 3, 3, "dimension %d does not have size 3", dimension + TH_INDEX_BASE); THTensor_(resizeAs)(r_, a); @@ -562,8 +562,8 @@ void THTensor_(diag)(THTensor *r_, THTensor *t, int k) if(THTensor_(nDimensionLegacyNoScalars)(t) == 1) { real *t_data = THTensor_(data)(t); - int64_t t_stride_0 = THTensor_(stride)(t, 0); - int64_t t_size = THTensor_(size)(t, 0); + int64_t t_stride_0 = THTensor_strideLegacyNoScalars(t, 0); + int64_t t_size = THTensor_sizeLegacyNoScalars(t, 0); int64_t sz = t_size + (k >= 0 ? k : -k); real *r__data; int64_t r__stride_0; @@ -1071,7 +1071,7 @@ void THTensor_(mode)(THTensor *values_, THLongTensor *indices_, THTensor *t, int THLongTensor_resize(indices_, dim, NULL); THLongStorage_free(dim); - t_size_dim = THTensor_(size)(t, dimension); + t_size_dim = THTensor_sizeLegacyNoScalars(t, dimension); temp_ = THTensor_(new)(); THTensor_(resize1d)(temp_, t_size_dim); @@ -1129,7 +1129,7 @@ void THTensor_(kthvalue)(THTensor *values_, THLongTensor *indices_, THTensor *t, int64_t t_size_dim; THArgCheck(dimension >= 0 && dimension < THTensor_(nDimensionLegacyAll)(t), 3, "dimension out of range"); - THArgCheck(k > 0 && k <= t->size(dimension), 2, "selected index out of range"); + THArgCheck(k > 0 && k <= THTensor_sizeLegacyNoScalars(t, dimension), 2, "selected index out of range"); int in_dims = THTensor_(nDimensionLegacyAll)(t); THTensor_(preserveReduceDimSemantics)(values_, in_dims, dimension, keepdim); @@ -1140,7 +1140,7 @@ void THTensor_(kthvalue)(THTensor *values_, THLongTensor *indices_, THTensor *t, THLongTensor_resize(indices_, dim, NULL); THLongStorage_free(dim); - t_size_dim = THTensor_(size)(t, dimension); + t_size_dim = THTensor_sizeLegacyNoScalars(t, dimension); temp_ = THTensor_(new)(); THTensor_(resize1d)(temp_, t_size_dim); @@ -1175,7 +1175,7 @@ void THTensor_(median)(THTensor *values_, THLongTensor *indices_, THTensor *t, i THArgCheck(dimension >= 0 && dimension < THTensor_(nDimensionLegacyAll)(t), 3, "dimension out of range"); - t_size_dim = THTensor_(size)(t, dimension); + t_size_dim = THTensor_sizeLegacyNoScalars(t, dimension); k = (t_size_dim-1) >> 1; /* take middle or one-before-middle element */ THTensor_(kthvalue)(values_, indices_, t, k+1, dimension, keepdim); @@ -1186,7 +1186,7 @@ void THTensor_(topk)(THTensor *rt_, THLongTensor *ri_, THTensor *t, int64_t k, i int numDims = THTensor_(nDimensionLegacyNoScalars)(t); THArgCheck(dim >= 0 && dim < numDims, 3, "dim not in range"); - int64_t sliceSize = THTensor_(size)(t, dim); + int64_t sliceSize = THTensor_sizeLegacyNoScalars(t, dim); THArgCheck(k >= 0 && k <= sliceSize, 2, "k not in range for dimension"); THTensor *tmpResults = THTensor_(new)(); @@ -1687,8 +1687,8 @@ void THTensor_(logicalAnd)(THTensor *r_, THTensor *t, int dimension, int keepdim real *t_data = tp+tBasicIndex; real *r__data = rp+iter; *r__data = 1; - for(j=0; j < t->size(dimension); ++j) { - *r__data = *r__data && *(t_data + j*t->stride(dimension)); + for(j=0; j < THTensor_sizeLegacyNoScalars(t, dimension); ++j) { + *r__data = *r__data && *(t_data + j*THTensor_strideLegacyNoScalars(t, dimension)); } } } else { @@ -1701,7 +1701,7 @@ void THTensor_(logicalAnd)(THTensor *r_, THTensor *t, int dimension, int keepdim if(serial_path) { // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { TH_TENSOR_DIM_APPLY2(real, t, real, r_, dimension, accreal prod = 1; int64_t i; @@ -1712,7 +1712,7 @@ void THTensor_(logicalAnd)(THTensor *r_, THTensor *t, int dimension, int keepdim THTensor_(fill)(r_, 1); THTensor *temp_ = THTensor_(newWithTensor)(r_); // r_.expand_as(t) - THTensor_setSizeAtDim(temp_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(temp_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(temp_, dimension, 0); TH_TENSOR_APPLY2(real, temp_, real, t, *temp__data = *temp__data && *t_data;); @@ -1767,8 +1767,8 @@ void THTensor_(logicalAny)(THTensor *r_, THTensor *t, int dimension, int keepdim real *t_data = tp+tBasicIndex; real *r__data = rp+iter; *r__data = 0; - for(j=0; j < t->size(dimension); ++j) { - *r__data = *r__data || *(t_data + j*t->stride(dimension)); + for(j=0; j < THTensor_sizeLegacyNoScalars(t, dimension); ++j) { + *r__data = *r__data || *(t_data + j*THTensor_strideLegacyNoScalars(t, dimension)); } } } else { @@ -1780,7 +1780,7 @@ void THTensor_(logicalAny)(THTensor *r_, THTensor *t, int dimension, int keepdim #endif if (serial_path) { // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { TH_TENSOR_DIM_APPLY2(real, t, real, r_, dimension, accreal sum = 0; int64_t i; @@ -1791,7 +1791,7 @@ void THTensor_(logicalAny)(THTensor *r_, THTensor *t, int dimension, int keepdim THTensor_(zero)(r_); THTensor *temp_ = THTensor_(newWithTensor)(r_); // r_.expand_as(t) - THTensor_setSizeAtDim(temp_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(temp_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(temp_, dimension, 0); TH_TENSOR_APPLY2(real, temp_, real, t, *temp__data = *temp__data || *t_data;); @@ -1876,7 +1876,7 @@ void THTensor_(mean)(THTensor *r_, THTensor *t, int dimension, int keepdim) dimension + TH_INDEX_BASE); THTensor_(sum)(r_, t, dimension, keepdim); - THTensor_(div)(r_, r_, t->size(dimension)); + THTensor_(div)(r_, r_, THTensor_sizeLegacyNoScalars(t, dimension)); } void THTensor_(std)(THTensor *r_, THTensor *t, int dimension, int biased, int keepdim) @@ -2054,7 +2054,7 @@ void THTensor_(renorm)(THTensor *res, THTensor *src, real value, int dimension, THTensor_(resizeAs)(res, src); - for (int64_t i = 0; i < src->size(dimension); i++) + for (int64_t i = 0; i < THTensor_sizeLegacyNoScalars(src, dimension); i++) { real norm = 0; real new_norm; @@ -2206,7 +2206,7 @@ void THTensor_(bhistc)(THTensor *hist, THTensor *tensor, int64_t nbins, real min real minval; real maxval; - THTensor_(resize2d)(hist, tensor->size(0), nbins); + THTensor_(resize2d)(hist, THTensor_sizeLegacyNoScalars(tensor, 0), nbins); THTensor_(zero)(hist); minval = minvalue; diff --git a/aten/src/THC/THCDeviceTensorUtils.cuh b/aten/src/THC/THCDeviceTensorUtils.cuh index 426bd1b6514c3..4a8da4a14c51c 100644 --- a/aten/src/THC/THCDeviceTensorUtils.cuh +++ b/aten/src/THC/THCDeviceTensorUtils.cuh @@ -58,8 +58,8 @@ toDeviceTensor(THCState* state, THCTensor* t) { IndexT strides[Dim]; for (int i = 0; i < Dim; ++i) { - int64_t size = THCTensor_size(state, t, i); - int64_t stride = THCTensor_stride(state, t, i); + int64_t size = THTensor_sizeLegacyNoScalars(t, i); + int64_t stride = THTensor_strideLegacyNoScalars(t, i); maxOffset += (size - 1) * stride; diff --git a/aten/src/THC/THCReduce.cuh b/aten/src/THC/THCReduce.cuh index 93adc5034f186..df1ad7c9aec3e 100644 --- a/aten/src/THC/THCReduce.cuh +++ b/aten/src/THC/THCReduce.cuh @@ -410,8 +410,8 @@ bool THC_reduceDim(THCState* state, int keepdim) { ptrdiff_t inElements = THCTensor_nElement(state, in); - int64_t reductionSize = THCTensor_size(state, in, dim); - int64_t reductionStride = THCTensor_stride(state, in, dim); + int64_t reductionSize = THTensor_sizeLegacyNoScalars(in, dim); + int64_t reductionStride = THTensor_strideLegacyNoScalars(in, dim); ptrdiff_t outElements = inElements / reductionSize; if (THCTensor_nDimensionLegacyAll(state, out) > MAX_CUTORCH_DIMS || diff --git a/aten/src/THC/THCTensor.cpp b/aten/src/THC/THCTensor.cpp index a8fb33c11a5bd..2ca851d9c2cad 100644 --- a/aten/src/THC/THCTensor.cpp +++ b/aten/src/THC/THCTensor.cpp @@ -22,10 +22,20 @@ int64_t THCTensor_size(THCState *state, const THCTensor *self, int dim) { return self->size(dim); } +int64_t THCTensor_sizeLegacyNoScalars(THCState *state, const THCTensor *self, int dim) { + return THTensor_sizeLegacyNoScalars(self, dim); +} + + int64_t THCTensor_stride(THCState *state, const THCTensor *self, int dim) { THArgCheck((dim >= 0) && (dim < self->dim()), 2, "out of range"); return self->stride(dim); } + +int64_t THCTensor_strideLegacyNoScalars(THCState *state, const THCTensor *self, int dim) { + return THTensor_strideLegacyNoScalars(self, dim); +} + THLongStorage *THCTensor_newSizeOf(THCState *state, THCTensor *self) { THLongStorage *size = THLongStorage_newWithSize(self->dim()); THLongStorage_rawCopy(size, THTensor_getSizePtr(self)); @@ -310,6 +320,9 @@ bool THCTensor_canUse32BitIndexMath(THCState* state, const THCTensor* t, ptrdiff if (elements >= max_elem) { return false; } + if (t->dim() == 0) { + return true; + } ptrdiff_t offset = 0; ptrdiff_t linearId = elements - 1; @@ -393,7 +406,7 @@ bool THCTensor_maybeOverlappingIndices(THCState* state, const THCTensor* t) { int dims = THCTensor_nDimensionLegacyAll(state, t); int nonSize1Dims = 0; for (int i = 0; i < dims; ++i) { - int64_t size = THCTensor_size(state, t, i); + int64_t size = THCTensor_sizeLegacyNoScalars(state, t, i); if (size > 1) { info[nonSize1Dims].size = size; diff --git a/aten/src/THC/THCTensor.hpp b/aten/src/THC/THCTensor.hpp index 3dbc7d0151cd5..2ed8cbf808bd1 100644 --- a/aten/src/THC/THCTensor.hpp +++ b/aten/src/THC/THCTensor.hpp @@ -16,7 +16,9 @@ THC_API int THCTensor_nDimensionLegacyNoScalars(THCState *state, const THCTensor THC_API int THCTensor_nDimensionLegacyAll(THCState *state, const THCTensor *self); THC_API int64_t THCTensor_size(THCState *state, const THCTensor *self, int dim); +THC_API int64_t THCTensor_sizeLegacyNoScalars(THCState *state, const THCTensor *self, int dim); THC_API int64_t THCTensor_stride(THCState *state, const THCTensor *self, int dim); +THC_API int64_t THCTensor_strideLegacyNoScalars(THCState *state, const THCTensor *self, int dim); THC_API THLongStorage *THCTensor_newSizeOf(THCState *state, THCTensor *self); THC_API THCTensor *THCTensor_new(THCState *state, at::ScalarType scalar_type); diff --git a/aten/src/THC/THCTensorMathReduce.cuh b/aten/src/THC/THCTensorMathReduce.cuh index b680595bc51e7..cc3c7cb499bea 100644 --- a/aten/src/THC/THCTensorMathReduce.cuh +++ b/aten/src/THC/THCTensorMathReduce.cuh @@ -300,13 +300,13 @@ __host__ void THCTensor_varOuterDim(THCState *state, TensorTypeK *tgt, TensorTyp // Treat all outer dimensions (i.e. dim < dimension) as one. unsigned num_orows = 1; for (int64_t dim = 0; dim < dimension; dim++) { - num_orows *= THCTensor_size(state, src, dim); + num_orows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } - unsigned row_size = THCTensor_size(state, src, dimension); + unsigned row_size = THCTensor_sizeLegacyNoScalars(state, src, dimension); // Treat all inner dimensions (i.e. dim > dimension) as one. unsigned num_irows = 1; for (unsigned dim = dimension + 1; dim < ndim; dim++) { - num_irows *= THCTensor_size(state, src, dim); + num_irows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } dim3 threads(min(512, num_irows)); @@ -446,9 +446,9 @@ __host__ void THCTensor_varInnermostDim(THCState *state, TensorTypeK *tgt, Tenso // Treat all outer dimensions as a single dimension. unsigned num_rows = 1; for (unsigned dim = 0; dim < ndim - 1; dim++) { - num_rows *= THCTensor_size(state, src, dim); + num_rows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } - unsigned row_size = THCTensor_size(state, src, ndim - 1); + unsigned row_size = THCTensor_sizeLegacyNoScalars(state, src, ndim - 1); // From limited testing, 16x32 seemed a good compromise for handling both long and short dimensions. dim3 threads(16, 32); @@ -518,12 +518,12 @@ THC_transformReduceOuterDimIndex(THCState *state, unsigned ndim = THCTensor_nDimensionLegacyAll(state, src); unsigned num_orows = 1; for (int64_t dim = 0; dim < rdim; dim++) { - num_orows *= THCTensor_size(state, src, dim); + num_orows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } - unsigned row_size = THCTensor_size(state, src, rdim); + unsigned row_size = THCTensor_sizeLegacyNoScalars(state, src, rdim); unsigned num_irows = 1; for (unsigned dim = rdim + 1; dim < ndim; dim++) { - num_irows *= THCTensor_size(state, src, dim); + num_irows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } dim3 threads(min(512, num_irows)); @@ -621,9 +621,9 @@ THC_transformReduceInnermostDimIndex(THCState *state, unsigned ndim = THCTensor_nDimensionLegacyAll(state, src); unsigned num_rows = 1; for (unsigned dim = 0; dim < ndim - 1; dim++) { - num_rows *= THCTensor_size(state, src, dim); + num_rows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } - unsigned row_size = THCTensor_size(state, src, ndim - 1); + unsigned row_size = THCTensor_sizeLegacyNoScalars(state, src, ndim - 1); dim3 threads(16, 32); dim3 grid(min(1024, THCCeilDiv(num_rows, threads.y))); diff --git a/aten/src/THC/THCTensorSort.cu b/aten/src/THC/THCTensorSort.cu index 6a5ee6e6864d0..25ce459c48de7 100644 --- a/aten/src/THC/THCTensorSort.cu +++ b/aten/src/THC/THCTensorSort.cu @@ -8,7 +8,7 @@ void THCudaLongTensor_fillSliceWithIndex(THCState* state, ptrdiff_t inElements = THCudaLongTensor_nElement(state, t); if (inElements > 0) { - int64_t sliceSize = THCudaLongTensor_size(state, t, dim); + int64_t sliceSize = THCudaLongTensor_sizeLegacyNoScalars(state, t, dim); ptrdiff_t numSlices = inElements / sliceSize; dim3 grid; diff --git a/aten/src/THC/THCTensorTypeUtils.cuh b/aten/src/THC/THCTensorTypeUtils.cuh index 94c3da69d90bf..b5dc6c547ba50 100644 --- a/aten/src/THC/THCTensorTypeUtils.cuh +++ b/aten/src/THC/THCTensorTypeUtils.cuh @@ -62,8 +62,8 @@ getTensorInfo(THCState* state, TensorType* t) { int dims = THCTensor_nDimensionLegacyNoScalars(state, t); for (int i = 0; i < dims; ++i) { - sz[i] = THCTensor_size(state, t, i); - st[i] = THCTensor_stride(state, t, i); + sz[i] = THTensor_sizeLegacyNoScalars(t, i); + st[i] = THTensor_strideLegacyNoScalars(t, i); } return TensorInfo( diff --git a/aten/src/THC/generic/THCTensor.cpp b/aten/src/THC/generic/THCTensor.cpp index 83dfe36c54fcc..fdf80565bd413 100644 --- a/aten/src/THC/generic/THCTensor.cpp +++ b/aten/src/THC/generic/THCTensor.cpp @@ -592,14 +592,14 @@ void THCTensor_(set1d)(THCState *state, THCTensor *tensor, int64_t x0, real valu { THArgCheck(THTensor_nDimensionLegacyNoScalars(tensor) == 1, 1, "tensor must have one dimension"); THArgCheck( (x0 >= 0) && (x0 < THTensor_sizeLegacyNoScalars(tensor, 0)), 2, "out of range"); - THCStorage_(set)(state, THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*tensor->stride(0), value); + THCStorage_(set)(state, THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*THTensor_strideLegacyNoScalars(tensor, 0), value); } real THCTensor_(get1d)(THCState *state, const THCTensor *tensor, int64_t x0) { THArgCheck(THTensor_nDimensionLegacyNoScalars(tensor) == 1, 1, "tensor must have one dimension"); THArgCheck( (x0 >= 0) && (x0 < THTensor_sizeLegacyNoScalars(tensor, 0)), 2, "out of range"); - return THCStorage_(get)(state, THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*tensor->stride(0)); + return THCStorage_(get)(state, THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*THTensor_strideLegacyNoScalars(tensor, 0)); } void THCTensor_(set2d)(THCState *state, THCTensor *tensor, int64_t x0, int64_t x1, real value) diff --git a/aten/src/THC/generic/THCTensorIndex.cu b/aten/src/THC/generic/THCTensorIndex.cu index 82f56f9946e47..111ce61e502f7 100644 --- a/aten/src/THC/generic/THCTensorIndex.cu +++ b/aten/src/THC/generic/THCTensorIndex.cu @@ -19,14 +19,14 @@ static ptrdiff_t THCTensor_(getSliceSize)(THCState *state, THCTensor *dst, ptrdiff_t dstSliceSize = 1; for (int d = 0; d < dstDims; d++) { if (d != dim) { - dstSliceSize *= dst->size(d); + dstSliceSize *= THTensor_sizeLegacyNoScalars(dst, d); } } if (src == nullptr) return dstSliceSize; THArgCheck(dim < srcDims, 3, "Indexing dim is out of bounds"); - THArgCheck(THCudaLongTensor_nElement(state, index) == src->size(dim), 4, + THArgCheck(THCudaLongTensor_nElement(state, index) == THTensor_sizeLegacyNoScalars(src, dim), 4, "length of src.size[dim] is not equal to length of indices"); ptrdiff_t srcSliceSize = 1; @@ -36,8 +36,8 @@ static ptrdiff_t THCTensor_(getSliceSize)(THCState *state, THCTensor *dst, for (int d = 0; d < srcDims; d++) { if (d != dim) { - srcSliceSize *= src->size(d); - if (!mismatch && dst->size(d) != src->size(d)) mismatch = true; + srcSliceSize *= THTensor_sizeLegacyNoScalars(src, d); + if (!mismatch && THTensor_sizeLegacyNoScalars(dst, d) != THTensor_sizeLegacyNoScalars(src, d)) mismatch = true; } } @@ -111,7 +111,7 @@ void THCTensor_(indexCopy)(THCState *state, THCTensor *dst, int dim, THCudaLongT // of the tensor `indices`. ptrdiff_t sliceSize = THCTensor_(getSliceSize)(state, dst, dim, indices, src); ptrdiff_t srcTotalSize = THCTensor_(nElement)(state, src); - int64_t dstCopyDimSize = THCTensor_(size)(state, dst, dim); + int64_t dstCopyDimSize = THCTensor_(sizeLegacyNoScalars)(state, dst, dim); ptrdiff_t numIndices = THCudaLongTensor_nElement(state, indices); if (sliceSize == 0) { @@ -300,7 +300,7 @@ void THCTensor_(indexAdd)(THCState *state, THCTensor *dst, int dim, THCudaLongTe // of the tensor `indices`. ptrdiff_t sliceSize = THCTensor_(getSliceSize)(state, dst, dim, indices, src); ptrdiff_t srcTotalSize = THCTensor_(nElement)(state, src); - int64_t dstAddDimSize = THCTensor_(size)(state, dst, dim); + int64_t dstAddDimSize = THCTensor_(sizeLegacyNoScalars)(state, dst, dim); ptrdiff_t numIndices = THCudaLongTensor_nElement(state, indices); if (sliceSize == 0) { @@ -422,7 +422,7 @@ void THCTensor_(indexFill)(THCState *state, THCTensor *dst, int dim, THCudaLongT ptrdiff_t sliceSize = THCTensor_(getSliceSize)(state, dst, dim, indices, nullptr); ptrdiff_t dstTotalSize = THCTensor_(nElement)(state, dst); - int64_t dstFillDimSize = THCTensor_(size)(state, dst, dim); + int64_t dstFillDimSize = THCTensor_(sizeLegacyNoScalars)(state, dst, dim); ptrdiff_t numIndices = THCudaLongTensor_nElement(state, indices); if (sliceSize == 0) { @@ -554,7 +554,7 @@ void THCTensor_(indexSelect)(THCState *state, THCTensor *dst, THCTensor *src, in // total size of the tensor ignoring dimension `dim`; // -the number of indices we are choosing, which is the total size // of the tensor `indices`. - int64_t srcSelectDimSize = THCTensor_(size)(state, src, dim); + int64_t srcSelectDimSize = THCTensor_(sizeLegacyNoScalars)(state, src, dim); ptrdiff_t sliceSize = dstTotalSize / numIndices; int mpc = THCState_getCurrentDeviceProperties(state)->multiProcessorCount; diff --git a/aten/src/THC/generic/THCTensorMath.cu b/aten/src/THC/generic/THCTensorMath.cu index cc1a8c9ba57e4..7fa0dbe1e190c 100644 --- a/aten/src/THC/generic/THCTensorMath.cu +++ b/aten/src/THC/generic/THCTensorMath.cu @@ -314,9 +314,9 @@ void THCTensor_(nonzero)(THCState* state, THCudaLongTensor *tensor, strided_tensor.begin(), strided_tensor.end(), stride_dim.begin(), - idx_functor(div, self->size(dim)) + idx_functor(div, THTensor_sizeLegacyNoScalars(self, dim)) ); - div *= self->size(dim); + div *= THTensor_sizeLegacyNoScalars(self, dim); } THCudaLongTensor_resize2d(state, tensor, num_nonzeros, num_dim); @@ -349,7 +349,7 @@ void THCTensor_(diag)(THCState *state, THCTensor *self_, THCTensor *src_, int64_ } else { ptrdiff_t totalElements = THCTensor_(nElement)(state, src_); ptrdiff_t size = (k > 0) ? totalElements + k : totalElements - k; - int64_t strideSrc = THCTensor_(stride)(state, src_, 0); + int64_t strideSrc = THTensor_strideLegacyNoScalars(src_, 0); THCTensor_(resize2d)(state, self_, size, size); THCTensor_(zero)(state, self_); if (size > 0) { diff --git a/aten/src/THC/generic/THCTensorMathBlas.cu b/aten/src/THC/generic/THCTensorMathBlas.cu index 591780b04edf7..0ab8165bf9443 100644 --- a/aten/src/THC/generic/THCTensorMathBlas.cu +++ b/aten/src/THC/generic/THCTensorMathBlas.cu @@ -63,7 +63,7 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real if(t->dim() != 1) THError("size mismatch"); - if(t->size(0) != mat->size(0)) + if(THTensor_sizeLegacyNoScalars(t, 0) != mat->size(0)) THError("size mismatch"); #if defined(THC_REAL_IS_FLOAT) || defined(THC_REAL_IS_DOUBLE) @@ -73,18 +73,20 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real THCTensor_(copy)(state, r_, t); } + auto r_stride = THTensor_strideLegacyNoScalars(r_, 0); + if(mat->stride(0) == 1) { #ifdef THC_REAL_IS_FLOAT THCudaBlas_Sgemv(state, 'n', mat->size(0), mat->size(1), alpha, THCTensor_(data)(state, mat), mat->stride(1), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #elif defined(THC_REAL_IS_DOUBLE) THCudaBlas_Dgemv(state, 'n', mat->size(0), mat->size(1), alpha, THCTensor_(data)(state, mat), mat->stride(1), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #endif } else if(mat->stride(1) == 1) @@ -93,12 +95,12 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real THCudaBlas_Sgemv(state, 't', mat->size(1), mat->size(0), alpha, THCTensor_(data)(state, mat), mat->stride(0), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #elif defined(THC_REAL_IS_DOUBLE) THCudaBlas_Dgemv(state, 't', mat->size(1), mat->size(0), alpha, THCTensor_(data)(state, mat), mat->stride(0), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #endif } else @@ -109,12 +111,12 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real THCudaBlas_Sgemv(state, 't', mat->size(1), mat->size(0), alpha, THCTensor_(data)(state, cmat), cmat->stride(0), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #elif defined(THC_REAL_IS_DOUBLE) THCudaBlas_Dgemv(state, 't', mat->size(1), mat->size(0), alpha, THCTensor_(data)(state, cmat), cmat->stride(0), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #endif THCTensor_(free)(state, cmat); @@ -122,7 +124,7 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real // In cublasSgemv, cublasDgemv (x,0).mv(0) does not // handle beta, whereas cublasSgemm, cublasDgemm do for case where (x,0).mm(0,y). - if (vec->size(0) == 0 && mat->size(0) != 0) { + if (THTensor_sizeLegacyNoScalars(vec, 0) == 0 && mat->size(0) != 0) { if(THCNumerics::eq(beta, ScalarConvert::to(0))) { THCTensor_(zero)(state, r_); } else if(THCNumerics::ne(beta, ScalarConvert::to(1))) { @@ -136,12 +138,12 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real THCTensor_(resize2d)(state, vecAsMatrix, vec_size, 1); THCTensor *tAsMatrix = THCTensor_(newWithTensor)(state, t); - THCTensor_(resize2d)(state, tAsMatrix, tAsMatrix->size(0), 1); + THCTensor_(resize2d)(state, tAsMatrix, THTensor_sizeLegacyNoScalars(tAsMatrix, 0), 1); THCTensor_(addmm)(state, r_, beta, tAsMatrix, alpha, mat, vecAsMatrix); // r_ will have answer as matrix, need to return a vector - THCTensor_(resize1d)(state, r_, r_->size(0)); + THCTensor_(resize1d)(state, r_, THTensor_sizeLegacyNoScalars(r_, 0)); THCTensor_(free)(state, vecAsMatrix); THCTensor_(free)(state, tAsMatrix); #endif @@ -192,7 +194,7 @@ THCTensor_(addr)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real a THCTensor_(data)(state, vec2), vec2_stride, THCTensor_(data)(state, r_), r_->stride(1)); #elif defined(THC_REAL_IS_DOUBLE) - THCudaBlas_Dger(state, vec1->size(0), vec2_size, + THCudaBlas_Dger(state, vec1_size, vec2_size, alpha, THCTensor_(data)(state, vec1), vec1_stride, THCTensor_(data)(state, vec2), vec2_stride, THCTensor_(data)(state, r_), r_->stride(1)); diff --git a/aten/src/THC/generic/THCTensorMathMagma.cu b/aten/src/THC/generic/THCTensorMathMagma.cu index dd2b2777d3552..aee04a8e22a4e 100644 --- a/aten/src/THC/generic/THCTensorMathMagma.cu +++ b/aten/src/THC/generic/THCTensorMathMagma.cu @@ -185,7 +185,7 @@ THC_API void THCTensor_(gels)(THCState *state, THCTensor *rb_, THCTensor *ra_, T THC_API void THCTensor_(syev)(THCState *state, THCTensor *re_, THCTensor *rv_, THCTensor *a, const char *jobzs, const char *uplos) { #ifdef USE_MAGMA - int64_t n = a->size(0); + int64_t n = THTensor_sizeLegacyNoScalars(a, 0); int64_t lda = n; magma_uplo_t uplo = uplos[0] == 'U' ? MagmaUpper : MagmaLower; diff --git a/aten/src/THC/generic/THCTensorMathPointwise.cu b/aten/src/THC/generic/THCTensorMathPointwise.cu index 46ddc50ffd693..a239da6635a15 100644 --- a/aten/src/THC/generic/THCTensorMathPointwise.cu +++ b/aten/src/THC/generic/THCTensorMathPointwise.cu @@ -114,9 +114,9 @@ THCTensor_(cross)(THCState *state, THCTensor *self, THCTensor *x, THCTensor *y, THCAssertSameGPU(THCTensor_(checkGPU)(state, 3, self, x, y)); int i; - int nd = THCTensor_(nDimensionLegacyNoScalars)(state, x); + int nd = x->dim(); ptrdiff_t nelem = THCTensor_(nElement)(state, x); - THArgCheck(nd == THCTensor_(nDimensionLegacyNoScalars)(state, y), 1, "tensors must have same number of dimensions"); + THArgCheck(nd == y->dim(), 1, "tensors must have same number of dimensions"); for (i = 0; i < nd; i++) { THArgCheck(THCTensor_(size)(state, x, i) == THCTensor_(size)(state, y, i), 1, "dimension %i of x and y does not match", i); if (dimension < 0 && THCTensor_(size)(state, x, i) == 3) { diff --git a/aten/src/THC/generic/THCTensorMathReduce.cu b/aten/src/THC/generic/THCTensorMathReduce.cu index 4300cc62ae4b4..614ce49d58274 100644 --- a/aten/src/THC/generic/THCTensorMathReduce.cu +++ b/aten/src/THC/generic/THCTensorMathReduce.cu @@ -68,8 +68,8 @@ THCTensor_(renorm)(THCState *state, THCTensor* self, THCTensor* src, real value, THArgCheck(THCTensor_(nDimensionLegacyNoScalars)(state, src) > 1, 1, "need at least 2 dimensions"); if (numel > 0) { - ptrdiff_t size = numel / data->size(0); - dim3 grid(data->size(0)); + ptrdiff_t size = numel / THTensor_sizeLegacyNoScalars(data, 0); + dim3 grid( THTensor_sizeLegacyNoScalars(data, 0)); dim3 threads(32); THCTensor_kernel_renorm diff --git a/aten/src/THC/generic/THCTensorMathScan.cu b/aten/src/THC/generic/THCTensorMathScan.cu index 3bb1c0bf1233a..708cb2ed4d63e 100644 --- a/aten/src/THC/generic/THCTensorMathScan.cu +++ b/aten/src/THC/generic/THCTensorMathScan.cu @@ -32,13 +32,13 @@ __host__ void THCTensor_(scanOuterDim)(THCState *state, THCTensor *tgt, // Treat all outer dimensions (i.e. dim < dimension) as one. unsigned num_orows = 1; for (int dim = 0; dim < dimension; dim++) { - num_orows *= THCTensor_(size)(state, src, dim); + num_orows *= THCTensor_(sizeLegacyNoScalars)(state, src, dim); } - unsigned row_size = THCTensor_(size)(state, src, dimension); + unsigned row_size = THCTensor_(sizeLegacyNoScalars)(state, src, dimension); // Treat all inner dimensions (i.e. dim > dimension) as one. unsigned num_irows = 1; for (unsigned dim = dimension + 1; dim < ndim; dim++) { - num_irows *= THCTensor_(size)(state, src, dim); + num_irows *= THCTensor_(sizeLegacyNoScalars)(state, src, dim); } dim3 threads(min(512, num_irows)); @@ -61,9 +61,9 @@ __host__ void THCTensor_(scanInnermostDim)(THCState *state, THCTensor *tgt, // Treat all outer dimensions as a single dimension. unsigned num_rows = 1; for (unsigned dim = 0; dim < ndim - 1; dim++) { - num_rows *= THCTensor_(size)(state, src, dim); + num_rows *= THCTensor_(sizeLegacyNoScalars)(state, src, dim); } - unsigned row_size = THCTensor_(size)(state, src, ndim - 1); + unsigned row_size = THCTensor_(sizeLegacyNoScalars)(state, src, ndim - 1); dim3 threads(16, 32); dim3 grid(min(1024, THCCeilDiv(num_rows, threads.y))); diff --git a/aten/src/THC/generic/THCTensorMode.cu b/aten/src/THC/generic/THCTensorMode.cu index a3c1cc8d067d7..1903995994d5f 100644 --- a/aten/src/THC/generic/THCTensorMode.cu +++ b/aten/src/THC/generic/THCTensorMode.cu @@ -17,10 +17,10 @@ THC_API void THCTensor_(calculateMode)(THCState *state, // calculations to get an offset real *data = THCTensor_(data)(state, input); for (int i = 0; i < THLongStorage_size(position); ++i) { - data += THLongStorage_data(position)[i] * THCTensor_(stride)(state, input, i); + data += THLongStorage_data(position)[i] * THTensor_strideLegacyNoScalars(input, i); } - int64_t nElement = THCTensor_(size)(state, input, THCTensor_(nDimensionLegacyAll)(state, input) - 1); + int64_t nElement = THCTensor_(sizeLegacyNoScalars)(state, input, THCTensor_(nDimensionLegacyAll)(state, input) - 1); THCThrustAllocator thrustAlloc(state); // Wrap input data, sortBuffer, in Thrust device vectors @@ -121,8 +121,8 @@ THC_API void THCTensor_(calculateMode)(THCState *state, for (int i = 0; i < THLongStorage_size(position); ++i) { int64_t pos = THLongStorage_data(position)[i]; - valuesOffset += THCTensor_(stride)(state, values, i) * pos; - indicesOffset += THCudaLongTensor_stride(state, indices, i) * pos; + valuesOffset += THTensor_strideLegacyNoScalars(values, i) * pos; + indicesOffset += THTensor_strideLegacyNoScalars(indices, i) * pos; } THCStorage_(set)(state, THCTensor_(storage)(state, values), valuesOffset, mode); THCudaLongStorage_set(state, THCudaLongTensor_storage(state, indices), indicesOffset, index); @@ -145,7 +145,7 @@ THC_API void THCTensor_(dimApplyMode)(THCState *state, THCTensor_(calculateMode)(state, values, indices, input, sortBuffer, dimension, position); } else { // Loop through the values and recurse - for (int i = 0; i < THCTensor_(size)(state, input, curDim); ++i) { + for (int i = 0; i < THCTensor_(sizeLegacyNoScalars)(state, input, curDim); ++i) { THLongStorage_data(position)[curDim] = i; THCTensor_(dimApplyMode)(state, values, indices, input, sortBuffer, dimension, position, curDim + 1); } @@ -175,7 +175,7 @@ THC_API void THCTensor_(mode)(THCState *state, ndim = THCTensor_(nDimensionLegacyAll)(state, input); THArgCheck(dimension >= 0 && dimension < ndim, 4, "Dimension of out bounds"); - sliceSize = THCTensor_(size)(state, input, dimension); + sliceSize = THCTensor_(sizeLegacyNoScalars)(state, input, dimension); slices = THCTensor_(nElement)(state, input) / sliceSize; // Resize output value, index Tensors to appropriate sizes (i.e. the same as diff --git a/aten/src/THC/generic/THCTensorRandom.cu b/aten/src/THC/generic/THCTensorRandom.cu index c03daad934beb..554b68fe9ea81 100644 --- a/aten/src/THC/generic/THCTensorRandom.cu +++ b/aten/src/THC/generic/THCTensorRandom.cu @@ -143,10 +143,10 @@ THC_API void THCTensor_(multinomial)(struct THCState *state, // Categories are in the innermost dimension int64_t numDist = - inputSize == 1 ? 1 : THCTensor_(size)(state, prob_dist, 0); + inputSize == 1 ? 1 : THCTensor_(sizeLegacyNoScalars)(state, prob_dist, 0); int64_t numCategoriesLong = - inputSize == 1 ? THCTensor_(size)(state, prob_dist, 0) : - THCTensor_(size)(state, prob_dist, 1); + inputSize == 1 ? THCTensor_(sizeLegacyNoScalars)(state, prob_dist, 0) : + THCTensor_(sizeLegacyNoScalars)(state, prob_dist, 1); // Since the index tensor is float, numCategories cannot exceed max // float integer precision diff --git a/aten/src/THC/generic/THCTensorScatterGather.cu b/aten/src/THC/generic/THCTensorScatterGather.cu index f8f75ef7edfb9..dd7d85f81dbc7 100644 --- a/aten/src/THC/generic/THCTensorScatterGather.cu +++ b/aten/src/THC/generic/THCTensorScatterGather.cu @@ -23,7 +23,7 @@ void THCTensor_(gather)(THCState* state, THCTensor *tensor, for (int d = 0; d < THCTensor_(nDimensionLegacyNoScalars)(state, tensor); d++) { if (d != dim) { - THArgCheck(THCTensor_(size)(state, tensor, d) == THCTensor_(size)(state, src, d), 2, + THArgCheck(THCTensor_(sizeLegacyNoScalars)(state, tensor, d) == THCTensor_(sizeLegacyNoScalars)(state, src, d), 2, "Input tensor must have same size as output tensor apart from the specified dimension"); } } @@ -115,13 +115,13 @@ void THCTensor_(scatter)(THCState* state, THCTensor *tensor, int dim, THCudaLong "Input tensor must have same dimensions as output tensor"); for (int d = 0; d < THCTensor_(nDimensionLegacyNoScalars)(state, tensor); d++) { - int64_t indexSizeD = THCudaLongTensor_size(state, index, d); + int64_t indexSizeD = THCudaLongTensor_sizeLegacyNoScalars(state, index, d); if (d != dim) { - THArgCheck(indexSizeD <= THCTensor_(size)(state, tensor, d), 3, + THArgCheck(indexSizeD <= THCTensor_(sizeLegacyNoScalars)(state, tensor, d), 3, "Index tensor must not have larger size than output tensor apart from the specified dimension %d, but got index %s output %s", dim, THCudaLongTensor_sizeDesc(state, index).str, THCTensor_(sizeDesc)(state, tensor).str); } - THArgCheck(indexSizeD <= THCTensor_(size)(state, src, d), 3, + THArgCheck(indexSizeD <= THCTensor_(sizeLegacyNoScalars)(state, src, d), 3, "Index tensor must not have larger size than input tensor, but got index %s input %s", THCudaLongTensor_sizeDesc(state, index).str, THCTensor_(sizeDesc)(state, src).str); } @@ -207,13 +207,13 @@ void THCTensor_(scatterAdd)(THCState* state, THCTensor *tensor, int dim, THCudaL "Input tensor must have same dimensions as output tensor"); for (int d = 0; d < THCTensor_(nDimensionLegacyNoScalars)(state, tensor); d++) { - int64_t indexSizeD = THCudaLongTensor_size(state, index, d); + int64_t indexSizeD = THCudaLongTensor_sizeLegacyNoScalars(state, index, d); if (d != dim) { - THArgCheck(indexSizeD <= THCTensor_(size)(state, tensor, d), 3, + THArgCheck(indexSizeD <= THCTensor_(sizeLegacyNoScalars)(state, tensor, d), 3, "Index tensor must not have larger size than output tensor apart from the specified dimension %d, but got index %s output %s", dim, THCudaLongTensor_sizeDesc(state, index).str, THCTensor_(sizeDesc)(state, tensor).str); } - THArgCheck(indexSizeD <= THCTensor_(size)(state, src, d), 3, + THArgCheck(indexSizeD <= THCTensor_(sizeLegacyNoScalars)(state, src, d), 3, "Index tensor must not have larger size than input tensor, but got index %s input %s", THCudaLongTensor_sizeDesc(state, index).str, THCTensor_(sizeDesc)(state, src).str); } @@ -301,8 +301,8 @@ THCTensor_(scatterFill)(THCState* state, THCTensor *tensor, for (int d = 0; d < THCTensor_(nDimensionLegacyNoScalars)(state, tensor); d++) { if (d != dim) { - THArgCheck(THCTensor_(size)(state, tensor, d) == - THCudaLongTensor_size(state, index, d), 4, + THArgCheck(THCTensor_(sizeLegacyNoScalars)(state, tensor, d) == + THCudaLongTensor_sizeLegacyNoScalars(state, index, d), 4, "Index tensor must have same size as output tensor apart from the specified dimension"); } } diff --git a/aten/src/THC/generic/THCTensorSort.cu b/aten/src/THC/generic/THCTensorSort.cu index e1bb29bd7dbbf..af81898357b8f 100644 --- a/aten/src/THC/generic/THCTensorSort.cu +++ b/aten/src/THC/generic/THCTensorSort.cu @@ -22,7 +22,7 @@ THC_API void THCTensor_(sortKeyValueInplace)(THCState* state, return; } - int64_t keySliceSize = THCTensor_(size)(state, key, dim); + int64_t keySliceSize = THCTensor_(sizeLegacyNoScalars)(state, key, dim); ptrdiff_t keySlices = inElements / keySliceSize; // The amount of shared memory and block size is based on @@ -159,8 +159,8 @@ void THCTensor_(sortViaThrust)(THCState* state, int nDims = THCTensor_(nDimensionLegacyAll)(state, input); ptrdiff_t totalElements = THCTensor_(nElement)(state, input); - int64_t sliceSize = THCTensor_(size)(state, input, dim); - int64_t sliceStride = THCTensor_(stride)(state, input, dim); + int64_t sliceSize = THCTensor_(sizeLegacyNoScalars)(state, input, dim); + int64_t sliceStride = THTensor_strideLegacyNoScalars(input, dim); // We perform a vectorized segmented sort in Thrust. // Say we are sorting a (2, 3) tensor. We have in flattened form: @@ -295,7 +295,7 @@ THC_API void THCTensor_(sort)(THCState* state, THLongStorage_free(inputSize); // How large are the slices that we are sorting? - int64_t sliceSize = THCTensor_(size)(state, input, dim); + int64_t sliceSize = THCTensor_(sizeLegacyNoScalars)(state, input, dim); // Workaround: // CUDA 8 uses more shared memory than 7.5 for bitonicSortKVInPlace, diff --git a/aten/src/THC/generic/THCTensorTopK.cu b/aten/src/THC/generic/THCTensorTopK.cu index 24f9a80849f22..c3b3c55b6ce2e 100644 --- a/aten/src/THC/generic/THCTensorTopK.cu +++ b/aten/src/THC/generic/THCTensorTopK.cu @@ -17,7 +17,7 @@ THC_API void THCTensor_(topk)(THCState* state, THArgCheck(dim >= 0 && dim < numDims, 6, "dim not in range"); - int64_t sliceSize = THCTensor_(size)(state, input_, dim); + int64_t sliceSize = THCTensor_(sizeLegacyNoScalars)(state, input_, dim); THArgCheck(k >= 0 && k <= sliceSize, 5, "k not in range for dimension"); THCTensor *input = THCTensor_(newContiguous)(state, input_); diff --git a/aten/src/THNN/generic/AbsCriterion.c b/aten/src/THNN/generic/AbsCriterion.c index 73552a2bc3b50..05f14773ef565 100644 --- a/aten/src/THNN/generic/AbsCriterion.c +++ b/aten/src/THNN/generic/AbsCriterion.c @@ -54,7 +54,7 @@ void THNN_(AbsCriterion_updateGradInput)( } THNN_CHECK_DIM_SIZE(gradOutput, 1, 0, 1); - real norm = (reduction == Reduction::ElementwiseMean ? 1./((real)THTensor_(nElement)(input)) : 1.) * THTensor_(fastGet1d)(gradOutput, 0); + real norm = (reduction == Reduction::ElementwiseMean ? 1./((real)THTensor_(nElement)(input)) : 1.) * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); TH_TENSOR_APPLY3(real, gradInput, real, input, real, target, *gradInput_data = (*input_data - *target_data) >= 0 ? norm : -norm; diff --git a/aten/src/THNN/generic/BCECriterion.c b/aten/src/THNN/generic/BCECriterion.c index f3f74ca108128..079493e305650 100644 --- a/aten/src/THNN/generic/BCECriterion.c +++ b/aten/src/THNN/generic/BCECriterion.c @@ -106,7 +106,7 @@ void THNN_(BCECriterion_updateGradInput)( TH_TENSOR_APPLY3(real, gradInput, real, input, real, target, real x = *input_data; real y = *target_data; - *gradInput_data = - norm * (y - x) / ((1. - x + EPS) * (x + EPS)) * THTensor_(fastGet1d)(gradOutput, 0); + *gradInput_data = - norm * (y - x) / ((1. - x + EPS) * (x + EPS)) * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); ); if(weights) diff --git a/aten/src/THNN/generic/ClassNLLCriterion.c b/aten/src/THNN/generic/ClassNLLCriterion.c index 7db0531d60d1e..c635a278d519d 100644 --- a/aten/src/THNN/generic/ClassNLLCriterion.c +++ b/aten/src/THNN/generic/ClassNLLCriterion.c @@ -37,14 +37,14 @@ void THNN_(ClassNLLCriterion_updateOutput)( int i; #pragma omp parallel for private(i) for (i = 0; i < batch_size; i++) { - int cur_target = THLongTensor_fastGet1d(target, i) - TH_INDEX_BASE; + int cur_target = THLongTensor_fastGetLegacy1dNoScalars(target, i) - TH_INDEX_BASE; if (cur_target >= 0 && cur_target < n_classes) { if (cur_target == ignore_index) { THTensor_(fastSet1d)(output, i, 0.0f); continue; } - real cur_weight = weights ? THTensor_(fastGet1d)(weights, cur_target) : 1.0f; + real cur_weight = weights ? THTensor_(fastGetLegacy1dNoScalars)(weights, cur_target) : 1.0f; THTensor_(fastSet1d)(output, i, -THTensor_(fastGet2d)(input, i, cur_target) * cur_weight); } else { int tmp = -1; @@ -151,12 +151,12 @@ void THNN_(ClassNLLCriterion_updateGradInput)( int i; #pragma omp parallel for private(i) for (i = 0; i < batch_size; i++) { - int cur_target = THLongTensor_fastGet1d(target, i) - TH_INDEX_BASE; + int cur_target = THLongTensor_fastGetLegacy1dNoScalars(target, i) - TH_INDEX_BASE; if (cur_target == ignore_index) { continue; } - real weight = weights ? THTensor_(fastGet1d)(weights, cur_target) : 1.0f; - THTensor_(fastSet2d)(gradInput, i, cur_target, -weight * THTensor_(fastGet1d)(gradOutput, i)); + real weight = weights ? THTensor_(fastGetLegacy1dNoScalars)(weights, cur_target) : 1.0f; + THTensor_(fastSet2d)(gradInput, i, cur_target, -weight * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, i)); } return; } diff --git a/aten/src/THNN/generic/DistKLDivCriterion.c b/aten/src/THNN/generic/DistKLDivCriterion.c index 823360812cae5..64dfb345fe762 100644 --- a/aten/src/THNN/generic/DistKLDivCriterion.c +++ b/aten/src/THNN/generic/DistKLDivCriterion.c @@ -57,7 +57,7 @@ void THNN_(DistKLDivCriterion_updateGradInput)( real norm = (reduction == Reduction::ElementwiseMean ? 1./((real)THTensor_(nElement)(input)) : 1.); TH_TENSOR_APPLY3(real, gradInput, real, input, real, target, - *gradInput_data = *target_data > 0 ? norm * (-*target_data) * THTensor_(fastGet1d)(gradOutput, 0) : 0; + *gradInput_data = *target_data > 0 ? norm * (-*target_data) * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0) : 0; ); } diff --git a/aten/src/THNN/generic/MultiLabelMarginCriterion.c b/aten/src/THNN/generic/MultiLabelMarginCriterion.c index cd0ecbb1e9df0..7a28ee1cadb76 100644 --- a/aten/src/THNN/generic/MultiLabelMarginCriterion.c +++ b/aten/src/THNN/generic/MultiLabelMarginCriterion.c @@ -234,7 +234,7 @@ void THNN_(MultiLabelMarginCriterion_updateGradInput)( THNN_CHECK_DIM_SIZE(gradOutput, 1, 0, 1); for (t = 0; t < nframe*dim; t++) { - gradInput_data[t] *= THTensor_(fastGet1d)(gradOutput, 0); + gradInput_data[t] *= THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); } } else @@ -244,7 +244,7 @@ void THNN_(MultiLabelMarginCriterion_updateGradInput)( { for (d = 0; d < dim; d++) { - gradInput_data[t * dim + d] *= THTensor_(fastGet1d)(gradOutput, t); + gradInput_data[t * dim + d] *= THTensor_(fastGetLegacy1dNoScalars)(gradOutput, t); } } } diff --git a/aten/src/THNN/generic/MultiMarginCriterion.c b/aten/src/THNN/generic/MultiMarginCriterion.c index 2c8f38be23eb3..92dbc520907ba 100644 --- a/aten/src/THNN/generic/MultiMarginCriterion.c +++ b/aten/src/THNN/generic/MultiMarginCriterion.c @@ -199,7 +199,7 @@ void THNN_(MultiMarginCriterion_updateGradInput)( { THNN_CHECK_DIM_SIZE(gradOutput, 1, 0, 1); for (t = 0; t < nframe * dim; t++) { - gradInput_data[t] *= THTensor_(fastGet1d)(gradOutput, 0); + gradInput_data[t] *= THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); } } else @@ -209,7 +209,7 @@ void THNN_(MultiMarginCriterion_updateGradInput)( { for (d = 0; d < dim; d++) { - gradInput_data[t * dim + d] *= THTensor_(fastGet1d)(gradOutput, t); + gradInput_data[t * dim + d] *= THTensor_(fastGetLegacy1dNoScalars)(gradOutput, t); } } } diff --git a/aten/src/THNN/generic/SmoothL1Criterion.c b/aten/src/THNN/generic/SmoothL1Criterion.c index b9eca659eeeb2..e8b2398483c20 100644 --- a/aten/src/THNN/generic/SmoothL1Criterion.c +++ b/aten/src/THNN/generic/SmoothL1Criterion.c @@ -64,7 +64,7 @@ void THNN_(SmoothL1Criterion_updateGradInput)( } THNN_CHECK_DIM_SIZE(gradOutput, 1, 0, 1); - real norm = (reduction == Reduction::ElementwiseMean ? 1./((real)THTensor_(nElement)(input)) : 1.) * THTensor_(fastGet1d)(gradOutput, 0); + real norm = (reduction == Reduction::ElementwiseMean ? 1./((real)THTensor_(nElement)(input)) : 1.) * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); TH_TENSOR_APPLY3(real, gradInput, real, input, real, target, real x = *input_data - *target_data; diff --git a/aten/src/THNN/generic/SoftMarginCriterion.c b/aten/src/THNN/generic/SoftMarginCriterion.c index 8fb31f9952e32..08c879e169e11 100644 --- a/aten/src/THNN/generic/SoftMarginCriterion.c +++ b/aten/src/THNN/generic/SoftMarginCriterion.c @@ -59,7 +59,7 @@ void THNN_(SoftMarginCriterion_updateGradInput)( TH_TENSOR_APPLY3(real, gradInput, real, input, real, target, real z = exp(-*target_data * *input_data); - *gradInput_data = -norm*(*target_data)*z/(1. + z) * THTensor_(fastGet1d)(gradOutput, 0);) + *gradInput_data = -norm*(*target_data)*z/(1. + z) * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0);) } #endif diff --git a/aten/src/THNN/generic/SpatialClassNLLCriterion.c b/aten/src/THNN/generic/SpatialClassNLLCriterion.c index 967f79c370924..246ba08998703 100644 --- a/aten/src/THNN/generic/SpatialClassNLLCriterion.c +++ b/aten/src/THNN/generic/SpatialClassNLLCriterion.c @@ -78,7 +78,7 @@ void THNN_(SpatialClassNLLCriterion_updateOutput)( continue; } real value = THTensor_(fastGet4d)(input, b, cur_target, h, w); - real weight = weights ? THTensor_(fastGet1d)(weights, cur_target) : 1.0f; + real weight = weights ? THTensor_(fastGetLegacy1dNoScalars)(weights, cur_target) : 1.0f; THTensor_(fastSet3d)(output, b, h, w, -value * weight); } } @@ -161,7 +161,7 @@ void THNN_(SpatialClassNLLCriterion_updateGradInput)( if (cur_target == ignore_index) { continue; } - real value = -(weights ? THTensor_(fastGet1d)(weights, cur_target) : 1.0f); + real value = -(weights ? THTensor_(fastGetLegacy1dNoScalars)(weights, cur_target) : 1.0f); real gradOutput_value = THTensor_(fastGet3d)(gradOutput, b, h, w); THTensor_(fastSet4d)(gradInput, b, cur_target, h, w, value * gradOutput_value); } @@ -201,7 +201,7 @@ void THNN_(SpatialClassNLLCriterion_updateGradInput)( int index = b * sample_size + cur_target * map_size + elem; gradInput_data[index] = - -(weights ? weights_data[cur_target] : 1.0f) / normalize * THTensor_(fastGet1d)(gradOutput, 0); + -(weights ? weights_data[cur_target] : 1.0f) / normalize * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); } } From 4a2f3cc45f2d7216a8a6598883a95dc7a7e46ff8 Mon Sep 17 00:00:00 2001 From: Lin Li Date: Thu, 2 Aug 2018 11:41:15 -0700 Subject: [PATCH 07/16] Improve lars operator by applying clipping (#9905) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/9905 This diff improves lars operator in Caffe2 by applying clipping to the computed learning rate Reviewed By: pjh5 Differential Revision: D9020606 fbshipit-source-id: b579f1d628113c09366feac9406002f1ef4bd54f --- caffe2/python/operator_test/lars_test.py | 24 +++++++++---- caffe2/python/optimizer.py | 30 ++++++++++++---- caffe2/sgd/lars_op.cc | 44 +++++++++++++++--------- caffe2/sgd/lars_op.h | 24 ++++++++++--- 4 files changed, 87 insertions(+), 35 deletions(-) diff --git a/caffe2/python/operator_test/lars_test.py b/caffe2/python/operator_test/lars_test.py index da983d22b85d4..e2f02b29d26f6 100755 --- a/caffe2/python/operator_test/lars_test.py +++ b/caffe2/python/operator_test/lars_test.py @@ -12,24 +12,34 @@ class TestLars(hu.HypothesisTestCase): - @given(offset=st.floats(min_value=0, max_value=100), **hu.gcs) - def test_lars(self, offset, dc, gc): + @given(offset=st.floats(min_value=0, max_value=100), + lr_min=st.floats(min_value=1e-8, max_value=1e-6), + **hu.gcs) + def test_lars(self, offset, lr_min, dc, gc): X = np.random.rand(6, 7, 8, 9).astype(np.float32) dX = np.random.rand(6, 7, 8, 9).astype(np.float32) + wd = np.array([1e-4]).astype(np.float32) + trust = np.random.rand(1).astype(np.float32) + lr_max = np.random.rand(1).astype(np.float32) - def ref_lars(X, dX): - return [1. / (np.linalg.norm(dX) / np.linalg.norm(X) + offset)] + def ref_lars(X, dX, wd, trust, lr_max): + rescale_factor = \ + trust / (np.linalg.norm(dX) / np.linalg.norm(X) + wd + offset) + rescale_factor = np.minimum(rescale_factor, lr_max) + rescale_factor = np.maximum(rescale_factor, lr_min) + return [rescale_factor] op = core.CreateOperator( "Lars", - ["X", "dX"], + ["X", "dX", "wd", "trust", "lr_max"], ["rescale_factor"], - offset=offset + offset=offset, + lr_min=lr_min, ) self.assertReferenceChecks( device_option=gc, op=op, - inputs=[X, dX], + inputs=[X, dX, wd, trust, lr_max], reference=ref_lars ) diff --git a/caffe2/python/optimizer.py b/caffe2/python/optimizer.py index ee60d776d55a8..2ebcf1d92a124 100644 --- a/caffe2/python/optimizer.py +++ b/caffe2/python/optimizer.py @@ -204,6 +204,14 @@ def scale_learning_rate(self, *args, **kwargs): raise NotImplementedError( "Optimizer Need to Implement `scale_learning_rate` method.") + def create_lars_inputs(self, param_init_net, weight_decay, trust, lr_max): + wd = param_init_net.ConstantFill([], "weight_decay", + shape=[1], value=weight_decay) + trust = param_init_net.ConstantFill([], "trust", shape=[1], value=trust) + lr_max = param_init_net.ConstantFill([], "lr_max", shape=[1], + value=lr_max) + return wd, trust, lr_max + class SgdOptimizer(Optimizer): def __init__(self, base_learning_rate=0.01, policy='fixed', @@ -233,10 +241,13 @@ def _run(self, net, param_init_net, param_info): if self.lars is not None and not isinstance(grad, core.GradientSlice): assert self.lars >= 0, ( 'Lars offset must be nonnegative, got {}'.format(self.lars)) + wd, trust, lr_max = self.create_lars_inputs( + param_init_net, 0.0, 1.0, np.finfo(np.float32).max) lr_lars_multiplier = net.Lars( - [param, grad], + [param, grad, wd, trust, lr_max], self.make_unique_blob_name(str(param) + "_lars"), - offset=self.lars) + offset=self.lars, + lr_min=0.0) current_scope = scope.CurrentDeviceScope() self._add_local_lr_multiplier( lr_lars_multiplier, @@ -520,10 +531,14 @@ def _run(self, net, param_init_net, param_info): if self.lars is not None and not isinstance(grad, core.GradientSlice): assert self.lars >= 0, ( 'Lars offset must be nonnegative, got {}'.format(self.lars)) + wd, trust, lr_max = self.create_lars_inputs( + param_init_net, 0.0, 1.0, np.finfo(np.float32).max) lr_lars_multiplier = net.Lars( - [param, grad], + [param, grad, wd, trust, lr_max], self.make_unique_blob_name(str(param) + "_lars"), - offset=self.lars) + offset=self.lars, + lr_min=0.0) + current_scope = scope.CurrentDeviceScope() self._add_local_lr_multiplier( lr_lars_multiplier, @@ -641,10 +656,13 @@ def _run(self, net, param_init_net, param_info): if self.lars is not None and not isinstance(grad, core.GradientSlice): assert self.lars >= 0, ( 'Lars offset must be nonnegative, got {}'.format(self.lars)) + wd, trust, lr_max = self.create_lars_inputs( + param_init_net, 0.0, 1.0, np.finfo(np.float32).max) lr_lars_multiplier = net.Lars( - [param, grad], + [param, grad, wd, trust, lr_max], self.make_unique_blob_name(str(param) + "_lars"), - offset=self.lars) + offset=self.lars, + lr_min=0.0) current_scope = scope.CurrentDeviceScope() self._add_local_lr_multiplier( lr_lars_multiplier, diff --git a/caffe2/sgd/lars_op.cc b/caffe2/sgd/lars_op.cc index 3e013a943464b..9ab418774e346 100644 --- a/caffe2/sgd/lars_op.cc +++ b/caffe2/sgd/lars_op.cc @@ -10,45 +10,55 @@ void LarsOp::Compute( TIndex N, const float* X_data, const float* dX_data, + const float* wd, + const float* trust, + const float* lr_max, float offset, - float* lr_rescale_data) { - *lr_rescale_data = 1.0; - + float lr_min, + float* lr_rescaled) { + float val = 1.0; float X_norm = sqrtf((ConstEigenVectorMap(X_data, N).array()).square().sum()); - if (X_norm > 0) { float dX_norm = sqrtf((ConstEigenVectorMap(dX_data, N).array()).square().sum()); - *lr_rescale_data /= (dX_norm / X_norm + offset); + val = (*trust) / (dX_norm / X_norm + (*wd) + offset); } + val = fmin(val, *lr_max); + val = fmax(val, lr_min); + *lr_rescaled = val; } REGISTER_CPU_OPERATOR(Lars, LarsOp); OPERATOR_SCHEMA(Lars) - .NumInputs(2) + .NumInputs(5) .NumOutputs(1) .SetDoc(R"DOC( -Implement Layer-wise Adaptive Rate Scaling (LARS) as in -https://arxiv.org/abs/1708.03888. Without weight decay, given a global -learning rate lr, parameter tensor X and its gradient dX, the local learning -rate for X will be +Implement Layer-wise Adaptive Rate Scaling (LARS) with clipping. Before adding weight +decay, given a parameter tensor X and its gradient dX, the local learning rate +for X will be - local_lr = lr * norm(X) / ( norm(dX) + offset * norm(X) ) +local_lr = trust * norm(X) / ( norm(dX) + wd * norm(X) + offset * norm(X) ) - = lr / ( norm(dX) / norm(X) + offset ), + = trust / ( norm(dX) / norm(X) + wd + offset ), -where offset is a preset hyper-parameter to avoid numerical issue. -In this implementation, we uses l2 norm and output the rescaling factor +where offset is a preset hyper-parameter to avoid numerical issue and trust +indicates how much we trust the layer to change its parameters during one update. +In this implementation, we uses l2 norm and the computed local learning rate is +clipped based on the upper bound lr_max and the lower bound lr_min: - 1 / ( norm(dX) / norm(X) + offset ). +local_lr = min(local_lr, lr_max) and local_lr = max(local_lr, lr_min) )DOC") .Input(0, "X", "Parameter tensor") .Input(1, "dX", "Gradient tensor") - .Output(0, "lr_rescale", "Local learning rate rescaling factor") - .Arg("offset", "rescaling offset parameter"); + .Input(2, "wd", "Weight decay") + .Input(3, "trust", "Trust") + .Input(4, "lr_max", "Upper bound of learning rate") + .Output(0, "lr_rescaled", "Rescaled local learning rate") + .Arg("offset", "rescaling offset parameter") + .Arg("lr_min", "minimum learning rate for clipping"); SHOULD_NOT_DO_GRADIENT(Lars); } // namespace caffe2 diff --git a/caffe2/sgd/lars_op.h b/caffe2/sgd/lars_op.h index f4eca35915ff7..0be04476fe2a2 100644 --- a/caffe2/sgd/lars_op.h +++ b/caffe2/sgd/lars_op.h @@ -13,7 +13,8 @@ class LarsOp final : public Operator { USE_OPERATOR_CONTEXT_FUNCTIONS; LarsOp(const OperatorDef& operator_def, Workspace* ws) : Operator(operator_def, ws), - offset_(OperatorBase::GetSingleArgument("offset", 0.5)) {} + offset_(OperatorBase::GetSingleArgument("offset", 0.5)), + lr_min_(OperatorBase::GetSingleArgument("lr_min", 0.02)) {} bool RunOnDevice() override { auto& X = Input(0); @@ -21,16 +22,24 @@ class LarsOp final : public Operator { CAFFE_ENFORCE( dX.size() == X.size(), "Gradient size doesn't match parameter size."); CAFFE_ENFORCE_GE(offset_, 0); + CAFFE_ENFORCE_GE(lr_min_, 0); - auto* lr_rescale = Output(0); - lr_rescale->Resize(vector{1}); + auto& wd = Input(2); + auto& trust = Input(3); + auto& lr_max = Input(4); + auto* lr_rescaled = Output(0); + lr_rescaled->Resize(vector{1}); Compute( dX.size(), X.template data(), dX.template data(), + wd.template data(), + trust.template data(), + lr_max.template data(), offset_, - lr_rescale->template mutable_data()); + lr_min_, + lr_rescaled->template mutable_data()); return true; } @@ -40,10 +49,15 @@ class LarsOp final : public Operator { TIndex N, const T* X_data, const T* dX_data, + const T* wd, + const T* trust, + const T* lr_max, T offset, - T* lr_rescale_data); + T lr_min, + T* lr_rescaled); T offset_; + T lr_min_; }; } // namespace caffe2 From 5765549155c86169a877d2e6f90dd03c791f5b84 Mon Sep 17 00:00:00 2001 From: Edward Yang Date: Thu, 2 Aug 2018 11:47:08 -0700 Subject: [PATCH 08/16] codemod -d caffe2 --extensions cc,h CaffeTypeId TypeIdentifier (#10166) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/10166 TypeIdentifier is still easy to codemod away from Reviewed By: smessmer Differential Revision: D9132840 fbshipit-source-id: bc83a8b17b2e7c19c9d2c9cfe5c7ce6ec1d8cec5 --- caffe2/core/blob_serialization.cc | 2 +- caffe2/core/blob_serialization.h | 4 +- caffe2/core/blob_stats.cc | 4 +- caffe2/core/blob_stats.h | 6 +-- caffe2/core/dispatch/DispatchKey.h | 6 +-- caffe2/core/plan_executor.cc | 2 +- caffe2/core/tensor.cc | 12 +++--- caffe2/core/tensor.h | 12 +++--- caffe2/core/typeid.cc | 18 ++++----- caffe2/core/typeid.h | 56 ++++++++++++++-------------- caffe2/core/types.cc | 2 +- caffe2/operators/quant_decode_op.h | 2 +- caffe2/python/pybind_state.cc | 4 +- caffe2/python/pybind_state.h | 6 +-- caffe2/python/pybind_state_dlpack.cc | 2 +- caffe2/python/pybind_state_dlpack.h | 2 +- caffe2/python/pybind_state_ideep.cc | 2 +- 17 files changed, 71 insertions(+), 71 deletions(-) diff --git a/caffe2/core/blob_serialization.cc b/caffe2/core/blob_serialization.cc index b870aa39067e6..df5ffbfce98f1 100644 --- a/caffe2/core/blob_serialization.cc +++ b/caffe2/core/blob_serialization.cc @@ -322,7 +322,7 @@ void TensorSerializer::StoreDeviceDetail( // The actual serialization registry objects. CAFFE_DEFINE_TYPED_REGISTRY( BlobSerializerRegistry, - CaffeTypeId, + TypeIdentifier, BlobSerializerBase, std::unique_ptr); diff --git a/caffe2/core/blob_serialization.h b/caffe2/core/blob_serialization.h index 18cb95d541b4b..8f2378886db42 100644 --- a/caffe2/core/blob_serialization.h +++ b/caffe2/core/blob_serialization.h @@ -26,13 +26,13 @@ constexpr auto kChunkIdSeparator = "#%"; // The Blob serialization registry and serializer creator functions. CAFFE_DECLARE_TYPED_REGISTRY( BlobSerializerRegistry, - CaffeTypeId, + TypeIdentifier, BlobSerializerBase, std::unique_ptr); #define REGISTER_BLOB_SERIALIZER(id, ...) \ CAFFE_REGISTER_TYPED_CLASS(BlobSerializerRegistry, id, __VA_ARGS__) // Creates an operator with the given operator definition. -inline unique_ptr CreateSerializer(CaffeTypeId id) { +inline unique_ptr CreateSerializer(TypeIdentifier id) { return BlobSerializerRegistry()->Create(id); } diff --git a/caffe2/core/blob_stats.cc b/caffe2/core/blob_stats.cc index 057e966aa319f..0c9d0e1bea26e 100644 --- a/caffe2/core/blob_stats.cc +++ b/caffe2/core/blob_stats.cc @@ -2,7 +2,7 @@ namespace caffe2 { -const BlobStatGetter* BlobStatRegistry::get(CaffeTypeId id) { +const BlobStatGetter* BlobStatRegistry::get(TypeIdentifier id) { auto it = map_.find(id); if (it == map_.end()) { return nullptr; @@ -16,7 +16,7 @@ BlobStatRegistry& BlobStatRegistry::instance() { } void BlobStatRegistry::doRegister( - CaffeTypeId id, + TypeIdentifier id, std::unique_ptr&& v) { // don't use CAFFE_ENFORCE_EQ to avoid static initialization order fiasco. if (map_.count(id) > 0) { diff --git a/caffe2/core/blob_stats.h b/caffe2/core/blob_stats.h index f4ef21d0cfb75..178bda0edd78b 100644 --- a/caffe2/core/blob_stats.h +++ b/caffe2/core/blob_stats.h @@ -15,8 +15,8 @@ struct BlobStatGetter { struct BlobStatRegistry { private: - std::unordered_map> map_; - void doRegister(CaffeTypeId id, std::unique_ptr&& v); + std::unordered_map> map_; + void doRegister(TypeIdentifier id, std::unique_ptr&& v); public: template @@ -27,7 +27,7 @@ struct BlobStatRegistry { } }; - const BlobStatGetter* get(CaffeTypeId id); + const BlobStatGetter* get(TypeIdentifier id); static BlobStatRegistry& instance(); }; diff --git a/caffe2/core/dispatch/DispatchKey.h b/caffe2/core/dispatch/DispatchKey.h index 6622f553b64f0..fe309b7f709b7 100644 --- a/caffe2/core/dispatch/DispatchKey.h +++ b/caffe2/core/dispatch/DispatchKey.h @@ -16,8 +16,8 @@ struct TensorParameterDispatchKey final { // note: This dispatch key structure is not final yet and will change. Don't rely on it. DeviceTypeId deviceTypeId; LayoutId layoutId; - // TODO Move this CaffeTypeId to c10 namespace - caffe2::CaffeTypeId dataType; + // TODO Move this TypeIdentifier to c10 namespace + caffe2::TypeIdentifier dataType; }; inline constexpr bool operator==(const TensorParameterDispatchKey& lhs, const TensorParameterDispatchKey& rhs) { return lhs.deviceTypeId == rhs.deviceTypeId && lhs.layoutId == rhs.layoutId && lhs.dataType == rhs.dataType; @@ -34,7 +34,7 @@ namespace std { struct hash { // TODO constexpr hashing size_t operator()(const c10::details::TensorParameterDispatchKey& obj) const { - return std::hash()(obj.deviceTypeId) ^ std::hash()(obj.layoutId) ^ std::hash()(obj.dataType); + return std::hash()(obj.deviceTypeId) ^ std::hash()(obj.layoutId) ^ std::hash()(obj.dataType); } }; } // namespace std diff --git a/caffe2/core/plan_executor.cc b/caffe2/core/plan_executor.cc index 1944874437d73..7e8490d700a67 100644 --- a/caffe2/core/plan_executor.cc +++ b/caffe2/core/plan_executor.cc @@ -100,7 +100,7 @@ std::function getContinuationTest( // if the blob doesn't exist or is not initiaized, return false inline bool getShouldStop(const Blob* b) { - if (!b || b->meta().id() == CaffeTypeId::uninitialized()) { // not exist or uninitialized + if (!b || b->meta().id() == TypeIdentifier::uninitialized()) { // not exist or uninitialized return false; } diff --git a/caffe2/core/tensor.cc b/caffe2/core/tensor.cc index bd6e6687087f2..ed00a4578d652 100644 --- a/caffe2/core/tensor.cc +++ b/caffe2/core/tensor.cc @@ -68,10 +68,10 @@ TypeMeta GetTensorType(const void* c) { } // TODO(jerryzh): Remove -static CaffeMap type_call_registry_{ +static CaffeMap type_call_registry_{ {TypeMeta::Id(), GetTensorType}}; -TypeCall GetTypeCallFunction(CaffeTypeId id) { +TypeCall GetTypeCallFunction(TypeIdentifier id) { auto f = type_call_registry_.find(id); if (f == type_call_registry_.end()) { return nullptr; @@ -79,7 +79,7 @@ TypeCall GetTypeCallFunction(CaffeTypeId id) { return f->second; } -void RegisterTypeCallFunction(CaffeTypeId id, TypeCall c) { +void RegisterTypeCallFunction(TypeIdentifier id, TypeCall c) { type_call_registry_[id] = c; } @@ -98,12 +98,12 @@ vector GetTensorInfo( } // since we only have one tensor, probably need to remove this at some point? -static CaffeMap tensor_info_call_registry_{ +static CaffeMap tensor_info_call_registry_{ {TypeMeta::Id(), GetTensorInfo}}; // TODO: Remove this code in a separate diff, since we only have one // GetTensorInfo function now -TensorInfoCall GetTensorInfoFunction(CaffeTypeId id) { +TensorInfoCall GetTensorInfoFunction(TypeIdentifier id) { auto f = tensor_info_call_registry_.find(id); if (f == tensor_info_call_registry_.end()) { return nullptr; @@ -111,7 +111,7 @@ TensorInfoCall GetTensorInfoFunction(CaffeTypeId id) { return f->second; } -void RegisterTensorInfoFunction(CaffeTypeId id, TensorInfoCall c) { +void RegisterTensorInfoFunction(TypeIdentifier id, TensorInfoCall c) { tensor_info_call_registry_[id] = c; } diff --git a/caffe2/core/tensor.h b/caffe2/core/tensor.h index 757876c1f51c6..2cf4ed8fdce03 100644 --- a/caffe2/core/tensor.h +++ b/caffe2/core/tensor.h @@ -473,7 +473,7 @@ class Tensor { Deleter d = nullptr) { meta_ = meta; CAFFE_ENFORCE_WITH_CALLER( - meta_.id() != CaffeTypeId::uninitialized(), + meta_.id() != TypeIdentifier::uninitialized(), "To share with a raw external pointer you need to have meta " "already set."); CAFFE_ENFORCE_WITH_CALLER( @@ -600,7 +600,7 @@ class Tensor { */ inline void* raw_mutable_data() { CAFFE_ENFORCE_WITH_CALLER( - meta_.id() != CaffeTypeId::uninitialized(), + meta_.id() != TypeIdentifier::uninitialized(), "Calling raw_mutable_data() without meta, but the current meta is " "of unknown type."); return raw_mutable_data(meta_); @@ -829,8 +829,8 @@ constexpr int k_limit_default_ = 1000; // Type call registry typedef TypeMeta (*TypeCall)(const void*); -TypeCall GetTypeCallFunction(CaffeTypeId id); -void RegisterTypeCallFunction(CaffeTypeId id, TypeCall c); +TypeCall GetTypeCallFunction(TypeIdentifier id); +void RegisterTypeCallFunction(TypeIdentifier id, TypeCall c); // Shape call registry typedef vector (*TensorInfoCall)( @@ -838,8 +838,8 @@ typedef vector (*TensorInfoCall)( bool* shares_data, size_t* capacity, DeviceOption* device); -TensorInfoCall GetTensorInfoFunction(CaffeTypeId id); -void RegisterTensorInfoFunction(CaffeTypeId id, TensorInfoCall c); +TensorInfoCall GetTensorInfoFunction(TypeIdentifier id); +void RegisterTensorInfoFunction(TypeIdentifier id, TensorInfoCall c); // resize helper function void TensorVectorResize( diff --git a/caffe2/core/typeid.cc b/caffe2/core/typeid.cc index ba81e3babc6d0..d4c5294f4b5d3 100644 --- a/caffe2/core/typeid.cc +++ b/caffe2/core/typeid.cc @@ -13,8 +13,8 @@ using std::string; namespace caffe2 { -std::unordered_map& gTypeNames() { - static std::unordered_map g_type_names; +std::unordered_map& gTypeNames() { + static std::unordered_map g_type_names; return g_type_names; } @@ -59,14 +59,14 @@ void TypeMeta::_ThrowRuntimeTypeLogicError(const std::string& msg) { CAFFE_THROW(msg); } -CaffeTypeId CaffeTypeId::createTypeId() { - static std::atomic counter( +TypeIdentifier TypeIdentifier::createTypeId() { + static std::atomic counter( TypeMeta::Id<_CaffeHighestPreallocatedTypeId>().underlyingId()); - const CaffeTypeId::underlying_type new_value = ++counter; - if (new_value == std::numeric_limits::max()) { - throw std::logic_error("Ran out of available type ids. If you need more than 2^16 CAFFE_KNOWN_TYPEs, we need to increase CaffeTypeId to use more than 16 bit."); + const TypeIdentifier::underlying_type new_value = ++counter; + if (new_value == std::numeric_limits::max()) { + throw std::logic_error("Ran out of available type ids. If you need more than 2^16 CAFFE_KNOWN_TYPEs, we need to increase TypeIdentifier to use more than 16 bit."); } - return CaffeTypeId(new_value); + return TypeIdentifier(new_value); } CAFFE_DEFINE_KNOWN_TYPE(Tensor); @@ -103,7 +103,7 @@ namespace { // intended to be only instantiated once here. struct UninitializedTypeNameRegisterer { UninitializedTypeNameRegisterer() { - gTypeNames()[CaffeTypeId::uninitialized()] = "nullptr (uninitialized)"; + gTypeNames()[TypeIdentifier::uninitialized()] = "nullptr (uninitialized)"; } }; static UninitializedTypeNameRegisterer g_uninitialized_type_name_registerer; diff --git a/caffe2/core/typeid.h b/caffe2/core/typeid.h index facea9fa64d2f..609c67a61dbf2 100644 --- a/caffe2/core/typeid.h +++ b/caffe2/core/typeid.h @@ -19,10 +19,10 @@ #include "ATen/core/IdWrapper.h" namespace caffe2 { -class CaffeTypeId; +class TypeIdentifier; } -std::ostream& operator<<(std::ostream& stream, caffe2::CaffeTypeId typeId); +std::ostream& operator<<(std::ostream& stream, caffe2::TypeIdentifier typeId); namespace caffe2 { @@ -30,43 +30,43 @@ class TypeMeta; /** * A type id is a unique id for a given C++ type. - * You need to register your types using CAFFE_KNOWN_TYPE(MyType) to be able to use CaffeTypeId with custom types. + * You need to register your types using CAFFE_KNOWN_TYPE(MyType) to be able to use TypeIdentifier with custom types. * This is for example used to store the dtype of tensors. */ -class CaffeTypeId final : public at::IdWrapper { +class TypeIdentifier final : public at::IdWrapper { public: - static CaffeTypeId createTypeId(); + static TypeIdentifier createTypeId(); - friend std::ostream& ::operator<<(std::ostream& stream, CaffeTypeId typeId); - friend bool operator<(CaffeTypeId lhs, CaffeTypeId rhs); + friend std::ostream& ::operator<<(std::ostream& stream, TypeIdentifier typeId); + friend bool operator<(TypeIdentifier lhs, TypeIdentifier rhs); // This is 8, because 0 is uint8_t (due to ScalarType BC constraint) - static constexpr CaffeTypeId uninitialized() { - return CaffeTypeId(8); + static constexpr TypeIdentifier uninitialized() { + return TypeIdentifier(8); } private: - constexpr explicit CaffeTypeId(uint16_t id): IdWrapper(id) {} + constexpr explicit TypeIdentifier(uint16_t id): IdWrapper(id) {} friend class TypeMeta; }; // Allow usage in std::map / std::set // TODO Disallow this and rather use std::unordered_map/set everywhere -inline bool operator<(CaffeTypeId lhs, CaffeTypeId rhs) { +inline bool operator<(TypeIdentifier lhs, TypeIdentifier rhs) { return lhs.underlyingId() < rhs.underlyingId(); } } -AT_DEFINE_HASH_FOR_IDWRAPPER(caffe2::CaffeTypeId) +AT_DEFINE_HASH_FOR_IDWRAPPER(caffe2::TypeIdentifier) -inline std::ostream& operator<<(std::ostream& stream, caffe2::CaffeTypeId typeId) { +inline std::ostream& operator<<(std::ostream& stream, caffe2::TypeIdentifier typeId) { return stream << typeId.underlyingId(); } namespace caffe2 { -std::unordered_map& gTypeNames(); +std::unordered_map& gTypeNames(); std::unordered_set& gRegisteredTypeNames(); // A utility function to demangle a function name. @@ -95,7 +95,7 @@ std::mutex& gTypeRegistrationMutex(); template struct TypeNameRegisterer { - TypeNameRegisterer(CaffeTypeId id, const std::string& literal_name) { + TypeNameRegisterer(TypeIdentifier id, const std::string& literal_name) { std::lock_guard guard(gTypeRegistrationMutex()); #ifdef __GXX_RTTI (void)literal_name; @@ -141,7 +141,7 @@ class TypeMeta { * type, use TypeMeta::Make(). */ TypeMeta() noexcept - : id_(CaffeTypeId::uninitialized()), itemsize_(0), ctor_(nullptr), copy_(nullptr), dtor_(nullptr) {} + : id_(TypeIdentifier::uninitialized()), itemsize_(0), ctor_(nullptr), copy_(nullptr), dtor_(nullptr) {} /** * Copy constructor. @@ -159,7 +159,7 @@ class TypeMeta { // TypeMeta can only be created by Make, making sure that we do not // create incorrectly mixed up TypeMeta objects. TypeMeta( - CaffeTypeId i, + TypeIdentifier i, size_t s, PlacementNew* ctor, TypedCopy* copy, @@ -176,7 +176,7 @@ class TypeMeta { /** * Returns the type id. */ - const CaffeTypeId& id() const noexcept { + const TypeIdentifier& id() const noexcept { return id_; } /** @@ -229,7 +229,7 @@ class TypeMeta { * is generated during run-time. Do NOT serialize the id for storage. */ template - CAFFE2_API static CaffeTypeId Id(); + CAFFE2_API static TypeIdentifier Id(); /** * Returns the item size of the type. This is equivalent to sizeof(T). @@ -356,7 +356,7 @@ class TypeMeta { } private: - CaffeTypeId id_; + TypeIdentifier id_; size_t itemsize_; PlacementNew* ctor_; TypedCopy* copy_; @@ -393,16 +393,16 @@ inline bool operator!=(const TypeMeta& lhs, const TypeMeta& rhs) noexcept { #ifdef _MSC_VER #define CAFFE_KNOWN_TYPE(T) \ template <> \ - CAFFE2_EXPORT CaffeTypeId TypeMeta::Id() { \ - static const CaffeTypeId type_id = CaffeTypeId::createTypeId(); \ + CAFFE2_EXPORT TypeIdentifier TypeMeta::Id() { \ + static const TypeIdentifier type_id = TypeIdentifier::createTypeId(); \ static TypeNameRegisterer registerer(type_id, #T); \ return type_id; \ } #else // _MSC_VER #define CAFFE_KNOWN_TYPE(T) \ template <> \ - CaffeTypeId TypeMeta::Id() { \ - static const CaffeTypeId type_id = CaffeTypeId::createTypeId(); \ + TypeIdentifier TypeMeta::Id() { \ + static const TypeIdentifier type_id = TypeIdentifier::createTypeId(); \ static TypeNameRegisterer registerer(type_id, #T); \ return type_id; \ } @@ -417,14 +417,14 @@ inline bool operator!=(const TypeMeta& lhs, const TypeMeta& rhs) noexcept { #ifdef _MSC_VER #define CAFFE_DECLARE_KNOWN_TYPE(PreallocatedId, T) \ template <> \ - inline CAFFE2_EXPORT CaffeTypeId TypeMeta::Id() { \ - return CaffeTypeId(PreallocatedId); \ + inline CAFFE2_EXPORT TypeIdentifier TypeMeta::Id() { \ + return TypeIdentifier(PreallocatedId); \ } #else // _MSC_VER #define CAFFE_DECLARE_KNOWN_TYPE(PreallocatedId, T) \ template <> \ - inline CaffeTypeId TypeMeta::Id() { \ - return CaffeTypeId(PreallocatedId); \ + inline TypeIdentifier TypeMeta::Id() { \ + return TypeIdentifier(PreallocatedId); \ } #endif diff --git a/caffe2/core/types.cc b/caffe2/core/types.cc index c7f3b6a31754d..e1c907a19a55a 100644 --- a/caffe2/core/types.cc +++ b/caffe2/core/types.cc @@ -13,7 +13,7 @@ CAFFE_KNOWN_TYPE(caffe2::float16); TensorProto::DataType TypeMetaToDataType(const TypeMeta& meta) { static_assert(sizeof(int) == 4, "int in this compiler does not equal to 4 bytes."); - static std::map data_type_map { + static std::map data_type_map { {TypeMeta::Id(), TensorProto_DataType_FLOAT}, {TypeMeta::Id(), TensorProto_DataType_INT32}, // BYTE does not have a type meta to proto mapping: we should diff --git a/caffe2/operators/quant_decode_op.h b/caffe2/operators/quant_decode_op.h index 8068b2e00510e..219d579f81a00 100644 --- a/caffe2/operators/quant_decode_op.h +++ b/caffe2/operators/quant_decode_op.h @@ -73,7 +73,7 @@ inline void DecodeGeneral( Tensor* outDecoded, bool resizeOnly) { const static std::map< - std::pair, + std::pair, std::function numpy_type_map{ + static std::map numpy_type_map{ {TypeMeta::Id(), NPY_BOOL}, {TypeMeta::Id(), NPY_DOUBLE}, {TypeMeta::Id(), NPY_FLOAT}, diff --git a/caffe2/python/pybind_state.h b/caffe2/python/pybind_state.h index 894c420afa94b..ed15bb17685a1 100644 --- a/caffe2/python/pybind_state.h +++ b/caffe2/python/pybind_state.h @@ -62,12 +62,12 @@ class BlobFeederBase { CAFFE2_EXPORT CAFFE_DECLARE_TYPED_REGISTRY( BlobFetcherRegistry, - CaffeTypeId, + TypeIdentifier, BlobFetcherBase, std::unique_ptr); #define REGISTER_BLOB_FETCHER(id, ...) \ CAFFE_REGISTER_TYPED_CLASS(BlobFetcherRegistry, id, __VA_ARGS__) -inline unique_ptr CreateFetcher(CaffeTypeId id) { +inline unique_ptr CreateFetcher(TypeIdentifier id) { return BlobFetcherRegistry()->Create(id); } @@ -168,7 +168,7 @@ class TensorFeeder : public BlobFeederBase { const auto npy_type = PyArray_TYPE(array); const TypeMeta& meta = NumpyTypeToCaffe(npy_type); CAFFE_ENFORCE( - meta.id() != CaffeTypeId::uninitialized(), + meta.id() != TypeIdentifier::uninitialized(), "This numpy data type is not supported: ", PyArray_TYPE(array), "."); diff --git a/caffe2/python/pybind_state_dlpack.cc b/caffe2/python/pybind_state_dlpack.cc index 1fea0a73c86f2..abfe6bb343d26 100644 --- a/caffe2/python/pybind_state_dlpack.cc +++ b/caffe2/python/pybind_state_dlpack.cc @@ -15,7 +15,7 @@ const DLDeviceType* CaffeToDLDeviceType(int device_type) { } const DLDataType* CaffeToDLType(const TypeMeta& meta) { - static std::map dl_type_map{ + static std::map dl_type_map{ {TypeMeta::Id(), DLDataType{0, 8, 1}}, {TypeMeta::Id(), DLDataType{0, 16, 1}}, {TypeMeta::Id(), DLDataType{0, 32, 1}}, diff --git a/caffe2/python/pybind_state_dlpack.h b/caffe2/python/pybind_state_dlpack.h index 37bf82e90bc30..45c074f929f4f 100644 --- a/caffe2/python/pybind_state_dlpack.h +++ b/caffe2/python/pybind_state_dlpack.h @@ -39,7 +39,7 @@ class DLPackWrapper { if (tensor->size() <= 0) { tensor->Resize(0); } - if (tensor->meta().id() == CaffeTypeId::uninitialized()) { + if (tensor->meta().id() == TypeIdentifier::uninitialized()) { // treat uninitialized tensor as float tensor tensor->template mutable_data(); } diff --git a/caffe2/python/pybind_state_ideep.cc b/caffe2/python/pybind_state_ideep.cc index 8b4b98d87714f..668c812cd8e1a 100644 --- a/caffe2/python/pybind_state_ideep.cc +++ b/caffe2/python/pybind_state_ideep.cc @@ -116,7 +116,7 @@ class IDeepFeeder : public BlobFeederBase { const auto npy_type = PyArray_TYPE(array); const TypeMeta& meta = NumpyTypeToCaffe(npy_type); CAFFE_ENFORCE( - meta.id() != CaffeTypeId::uninitialized(), + meta.id() != TypeIdentifier::uninitialized(), "This numpy data type is not supported: ", PyArray_TYPE(array), "."); From ee98533746722dd2726a8cf149aa72d7cabda2f1 Mon Sep 17 00:00:00 2001 From: Junjie Bai Date: Thu, 2 Aug 2018 12:02:18 -0700 Subject: [PATCH 09/16] Fix compiler warnings on ignored const qualifiers Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/10142 Reviewed By: yinghai Differential Revision: D9125502 Pulled By: bddppq fbshipit-source-id: 8043b2a05507a4707220fa820ab6cc486760a93e --- caffe2/core/nomnigraph/include/nomnigraph/Graph/Graph.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caffe2/core/nomnigraph/include/nomnigraph/Graph/Graph.h b/caffe2/core/nomnigraph/include/nomnigraph/Graph/Graph.h index aab127d8c56e1..cb7b90059a156 100644 --- a/caffe2/core/nomnigraph/include/nomnigraph/Graph/Graph.h +++ b/caffe2/core/nomnigraph/include/nomnigraph/Graph/Graph.h @@ -189,7 +189,7 @@ class Subgraph { return nodes_; } - const size_t getNodesCount() const { + size_t getNodesCount() const { return (size_t)nodes_.size(); } From 6e85112f12cce9731ae09237769f15f675f3da7b Mon Sep 17 00:00:00 2001 From: Rob Kunkle Date: Thu, 2 Aug 2018 12:19:18 -0700 Subject: [PATCH 10/16] Adding katex rendering of equations, and required edits to equations. (#8848) Summary: This fixes issue #8529. - Adds Katex extension to conf.py and requirements.txt - Fixes syntax differences in docs - Should allow documentation pages to render faster Pull Request resolved: https://github.com/pytorch/pytorch/pull/8848 Reviewed By: soumith Differential Revision: D8677702 Pulled By: goodlux fbshipit-source-id: c4a832c5879e0eebcb14763b35a41663331ba23f --- docs/requirements.txt | 1 + docs/source/conf.py | 52 ++++- docs/source/nn.rst | 2 +- torch/_torch_docs.py | 16 +- torch/functional.py | 20 +- torch/nn/functional.py | 2 +- torch/nn/init.py | 10 +- torch/nn/modules/activation.py | 111 ++++++---- torch/nn/modules/conv.py | 98 ++++----- torch/nn/modules/fold.py | 30 +-- torch/nn/modules/linear.py | 14 +- torch/nn/modules/loss.py | 52 +++-- torch/nn/modules/normalization.py | 4 +- torch/nn/modules/padding.py | 345 +++++++++++++----------------- torch/nn/modules/pixelshuffle.py | 4 +- torch/nn/modules/pooling.py | 94 ++++---- torch/nn/modules/rnn.py | 31 ++- torch/nn/modules/upsampling.py | 24 ++- torch/nn/utils/spectral_norm.py | 4 +- 19 files changed, 476 insertions(+), 438 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 0c3ee1922efaa..159253ca0a6e3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ sphinx -e git://github.com/snide/sphinx_rtd_theme.git#egg=sphinx_rtd_theme +sphinxcontrib.katex diff --git a/docs/source/conf.py b/docs/source/conf.py index 1eaaa3b9086d9..1b4d8d6ff8319 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -19,7 +19,10 @@ # import os # import sys -# sys.path.insert(0, os.path.abspath('.')) + +# source code directory, relative to this file, for sphinx-autobuild +# sys.path.insert(0, os.path.abspath('../..')) + import torch try: import torchvision @@ -47,11 +50,40 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', 'sphinx.ext.napoleon', 'sphinx.ext.viewcode', + 'sphinxcontrib.katex', ] +# katex (mathjax replacement) macros +# +# + +katex_macros = r''' +"\\op": "\\operatorname{{#1}}", +"\\i": "\\mathrm{i}", +"\\e": "\\mathrm{e}^{#1}", +"\\w": "\\omega", +"\\vec": "\\mathbf{#1}", +"\\x": "\\vec{x}", +"\\d": "\\operatorname{d}\\!{}", +"\\dirac": "\\operatorname{\\delta}\\left(#1\\right)", +"\\scalarprod": "\\left\\langle#1,#2\\right\\rangle", +''' + +# katex options +# +# + +katex_options = r''' +delimiters : [ + {left: "$$", right: "$$", display: true}, + {left: "\\(", right: "\\)", display: true}, + {left: "\\[", right: "\\]", display: true} +], +strict : false +''' + napoleon_use_ivar = True # Add any paths that contain templates here, relative to this directory. @@ -106,14 +138,23 @@ autodoc_inherit_docstrings = False -# -- Options for HTML output ---------------------------------------------- +# -- katex javascript in header +# +# def setup(app): +# app.add_javascript("https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.js") + +# -- Options for HTML output ---------------------------------------------- +# # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # +# +# html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. @@ -135,11 +176,12 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static', '_images'] -# html_style_path = 'css/pytorch_theme.css' +html_style_path = 'css/pytorch_theme.css' html_context = { 'css_files': [ 'https://fonts.googleapis.com/css?family=Lato', - '_static/css/pytorch_theme.css' + '_static/css/pytorch_theme.css', + 'https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css', ], } diff --git a/docs/source/nn.rst b/docs/source/nn.rst index 283409ea3676b..9b3563f83b2d4 100644 --- a/docs/source/nn.rst +++ b/docs/source/nn.rst @@ -1120,7 +1120,7 @@ Linear functions .. autofunction:: linear :hidden:`bilinear` -~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~ .. autofunction:: bilinear diff --git a/torch/_torch_docs.py b/torch/_torch_docs.py index fad659a4c7145..91cfaff303c0b 100644 --- a/torch/_torch_docs.py +++ b/torch/_torch_docs.py @@ -5715,7 +5715,7 @@ def parse_kwargs(desc): window trims off the last duplicate value from the symmetric window and is ready to be used as a periodic window with functions like :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in -above formula is in fact :math:`\text{window_length} + 1`. Also, we always have +above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have ``torch.hann_window(L, periodic=True)`` equal to ``torch.hann_window(L + 1, periodic=False)[:-1])``. @@ -5733,7 +5733,7 @@ def parse_kwargs(desc): {requires_grad} Returns: - Tensor: A 1-D tensor of size :math:`(\text{{window_length}},)` containing the window + Tensor: A 1-D tensor of size :math:`(\text{{window\_length}},)` containing the window """.format(**factory_common_args)) @@ -5755,7 +5755,7 @@ def parse_kwargs(desc): window trims off the last duplicate value from the symmetric window and is ready to be used as a periodic window with functions like :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in -above formula is in fact :math:`\text{window_length} + 1`. Also, we always have +above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have ``torch.hamming_window(L, periodic=True)`` equal to ``torch.hamming_window(L + 1, periodic=False)[:-1])``. @@ -5776,7 +5776,7 @@ def parse_kwargs(desc): {requires_grad} Returns: - Tensor: A 1-D tensor of size :math:`(\text{{window_length}},)` containing the window + Tensor: A 1-D tensor of size :math:`(\text{{window\_length}},)` containing the window """.format(**factory_common_args)) @@ -5801,7 +5801,7 @@ def parse_kwargs(desc): window trims off the last duplicate value from the symmetric window and is ready to be used as a periodic window with functions like :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in -above formula is in fact :math:`\text{window_length} + 1`. Also, we always have +above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have ``torch.bartlett_window(L, periodic=True)`` equal to ``torch.bartlett_window(L + 1, periodic=False)[:-1])``. @@ -5819,7 +5819,7 @@ def parse_kwargs(desc): {requires_grad} Returns: - Tensor: A 1-D tensor of size :math:`(\text{{window_length}},)` containing the window + Tensor: A 1-D tensor of size :math:`(\text{{window\_length}},)` containing the window """.format(**factory_common_args)) @@ -5841,7 +5841,7 @@ def parse_kwargs(desc): window trims off the last duplicate value from the symmetric window and is ready to be used as a periodic window with functions like :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in -above formula is in fact :math:`\text{window_length} + 1`. Also, we always have +above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have ``torch.blackman_window(L, periodic=True)`` equal to ``torch.blackman_window(L + 1, periodic=False)[:-1])``. @@ -5859,7 +5859,7 @@ def parse_kwargs(desc): {requires_grad} Returns: - Tensor: A 1-D tensor of size :math:`(\text{{window_length}},)` containing the window + Tensor: A 1-D tensor of size :math:`(\text{{window\_length}},)` containing the window """.format(**factory_common_args)) diff --git a/torch/functional.py b/torch/functional.py index 0133a01298185..116238bbea79a 100644 --- a/torch/functional.py +++ b/torch/functional.py @@ -209,12 +209,12 @@ def stft(input, n_fft, hop_length=None, win_length=None, window=None, expression: .. math:: - X[m, \omega] = \sum_{k = 0}^{\text{win_length}}% + X[m, \omega] = \sum_{k = 0}^{\text{win\_length}}% window[k]\ input[m \times hop_length + k]\ % - e^{- j \frac{2 \pi \cdot \omega k}{\text{win_length}}}, + e^{- j \frac{2 \pi \cdot \omega k}{\text{win\_length}}}, where :math:`m` is the index of the sliding window, and :math:`\omega` is - the frequency that :math:`0 \leq \omega < \text{n_fft}`. When + the frequency that :math:`0 \leq \omega < \text{n\_fft}`. When :attr:`onesided` is the default value ``True``, * :attr:`input` must be either a 1-D time sequenceor 2-D a batch of time @@ -229,25 +229,25 @@ def stft(input, n_fft, hop_length=None, win_length=None, window=None, * :attr:`window` can be a 1-D tensor of size :attr:`win_length`, e.g., from :meth:`torch.hann_window`. If :attr:`window` is ``None`` (default), it is treated as if having :math:`1` everywhere in the window. If - :math:`\text{win_length} < \text{n_fft}`, :attr:`window` will be padded on + :math:`\text{win\_length} < \text{n\_fft}`, :attr:`window` will be padded on both sides to length :attr:`n_fft` before being applied. * If :attr:`center` is ``True`` (default), :attr:`input` will be padded on both sides so that the :math:`t`-th frame is centered at time - :math:`t \times \text{hop_length}`. Otherwise, the :math:`t`-th frame - begins at time :math:`t \times \text{hop_length}`. + :math:`t \times \text{hop\_length}`. Otherwise, the :math:`t`-th frame + begins at time :math:`t \times \text{hop\_length}`. * :attr:`pad_mode` determines the padding method used on :attr:`input` when :attr:`center` is ``True``. See :meth:`torch.nn.functional.pad` for all available options. Default is ``"reflect"``. * If :attr:`onesided` is ``True`` (default), only values for :math:`\omega` - in :math:`\left[0, 1, 2, \dots, \left\lfloor \frac{\text{n_fft}}{2} \right\rfloor + 1\right]` + in :math:`\left[0, 1, 2, \dots, \left\lfloor \frac{\text{n\_fft}}{2} \right\rfloor + 1\right]` are returned because the real-to-complex Fourier transform satisfies the - conjugate symmetry, i.e., :math:`X[m, \omega] = X[m, \text{n_fft} - \omega]^*`. + conjugate symmetry, i.e., :math:`X[m, \omega] = X[m, \text{n\_fft} - \omega]^*`. * If :attr:`normalized` is ``True`` (default is ``False``), the function - returns the normalized STFT results, i.e., multiplied by :math:`(\text{frame_length})^{-0.5}`. + returns the normalized STFT results, i.e., multiplied by :math:`(\text{frame\_length})^{-0.5}`. Returns the real and the imaginary parts together as one tensor of size :math:`(* \times N \times T \times 2)`, where :math:`*` is the optional @@ -270,7 +270,7 @@ def stft(input, n_fft, hop_length=None, win_length=None, window=None, window (Tensor, optional): the optional window function. Default: ``None`` (treated as window of all :math:`1`s) center (bool, optional): whether to pad :attr:`input` on both sides so - that the :math:`t`-th frame is centered at time :math:`t \times \text{hop_length}`. + that the :math:`t`-th frame is centered at time :math:`t \times \text{hop\_length}`. Default: ``True`` pad_mode (string, optional): controls the padding method used when :attr:`center` is ``True``. Default: ``"reflect"`` diff --git a/torch/nn/functional.py b/torch/nn/functional.py index 746c266452917..21c09c412af6f 100644 --- a/torch/nn/functional.py +++ b/torch/nn/functional.py @@ -765,7 +765,7 @@ def leaky_relu(input, negative_slope=0.01, inplace=False): leaky_relu(input, negative_slope=0.01, inplace=False) -> Tensor Applies element-wise, - :math:`\text{LeakyReLU}(x) = \max(0, x) + \text{negative_slope} * \min(0, x)` + :math:`\text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)` See :class:`~torch.nn.LeakyReLU` for more details. """ diff --git a/torch/nn/init.py b/torch/nn/init.py index ecc92606ecfbe..a5c8239d3dc96 100644 --- a/torch/nn/init.py +++ b/torch/nn/init.py @@ -17,7 +17,7 @@ def calculate_gain(nonlinearity, param=None): Sigmoid :math:`1` Tanh :math:`\frac{5}{3}` ReLU :math:`\sqrt{2}` - Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative_slope}^2}}` + Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}` ================= ==================================================== Args: @@ -203,7 +203,7 @@ def xavier_uniform_(tensor, gain=1): :math:`\mathcal{U}(-a, a)` where .. math:: - a = \text{gain} \times \sqrt{\frac{6}{\text{fan_in} + \text{fan_out}}} + a = \text{gain} \times \sqrt{\frac{6}{\text{fan\_in} + \text{fan\_out}}} Also known as Glorot initialization. @@ -230,7 +230,7 @@ def xavier_normal_(tensor, gain=1): :math:`\mathcal{N}(0, \text{std})` where .. math:: - \text{std} = \text{gain} \times \sqrt{\frac{2}{\text{fan_in} + \text{fan_out}}} + \text{std} = \text{gain} \times \sqrt{\frac{2}{\text{fan\_in} + \text{fan\_out}}} Also known as Glorot initialization. @@ -266,7 +266,7 @@ def kaiming_uniform_(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'): :math:`\mathcal{U}(-\text{bound}, \text{bound})` where .. math:: - \text{bound} = \sqrt{\frac{6}{(1 + a^2) \times \text{fan_in}}} + \text{bound} = \sqrt{\frac{6}{(1 + a^2) \times \text{fan\_in}}} Also known as He initialization. @@ -301,7 +301,7 @@ def kaiming_normal_(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'): :math:`\mathcal{N}(0, \text{std})` where .. math:: - \text{std} = \sqrt{\frac{2}{(1 + a^2) \times \text{fan_in}}} + \text{std} = \sqrt{\frac{2}{(1 + a^2) \times \text{fan\_in}}} Also known as He initialization. diff --git a/torch/nn/modules/activation.py b/torch/nn/modules/activation.py index 51cfab7940414..9c70d19ddd57b 100644 --- a/torch/nn/modules/activation.py +++ b/torch/nn/modules/activation.py @@ -82,17 +82,19 @@ def extra_repr(self): class RReLU(Module): - r"""Applies the randomized leaky rectified liner unit function element-wise - described in the paper + r"""Applies the randomized leaky rectified liner unit function, element-wise, + as described in the paper: + `Empirical Evaluation of Rectified Activations in Convolutional Network`_. The function is defined as: .. math:: - \text{RReLU}(x) = \begin{cases} + \text{RReLU}(x) = + \begin{cases} x & \text{if } x \geq 0 \\ ax & \text{ otherwise } - \end{cases}, + \end{cases} where :math:`a` is randomly sampled from uniform distribution :math:`\mathcal{U}(\text{lower}, \text{upper})`. @@ -195,7 +197,10 @@ def extra_repr(self): class ReLU6(Hardtanh): - r"""Applies the element-wise function :math:`\text{ReLU6}(x) = \min(\max(0,x), 6)` + r"""Applies the element-wise function: + + .. math:: + \text{ReLU6}(x) = \min(\max(0,x), 6) Args: inplace: can optionally do the operation in-place. Default: ``False`` @@ -223,7 +228,11 @@ def extra_repr(self): class Sigmoid(Module): - r"""Applies the element-wise function :math:`\text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}` + r"""Applies the element-wise function: + + .. math:: + \text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)} + Shape: - Input: :math:`(N, *)` where `*` means, any number of additional @@ -244,8 +253,10 @@ def forward(self, input): class Tanh(Module): - r"""Applies element-wise, - :math:`\text{Tanh}(x) = \tanh(x) = \frac{e^x - e^{-x}} {e^x + e^{-x}}` + r"""Applies the element-wise function: + + .. math:: + \text{Tanh}(x) = \tanh(x) = \frac{e^x - e^{-x}} {e^x + e^{-x}} Shape: - Input: :math:`(N, *)` where `*` means, any number of additional @@ -266,8 +277,10 @@ def forward(self, input): class ELU(Module): - r"""Applies element-wise, - :math:`\text{ELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1))` + r"""Applies the element-wise function: + + .. math:: + \text{ELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1)) Args: alpha: the :math:`\alpha` value for the ELU formulation. Default: 1.0 @@ -301,8 +314,10 @@ def extra_repr(self): class CELU(Module): - r"""Applies element-wise, - :math:`\text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))` + r"""Applies the element-wise function: + + .. math:: + \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1)) More details can be found in the paper `Continuously Differentiable Exponential Linear Units`_ . @@ -341,8 +356,11 @@ def extra_repr(self): class SELU(Module): - r"""Applies element-wise, - :math:`\text{SELU}(x) = \text{scale} * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))`, + r"""Applied element-wise, as: + + .. math:: + \text{SELU}(x) = \text{scale} * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1))) + with :math:`\alpha = 1.6732632423543772848170429916717` and :math:`\text{scale} = 1.0507009873554804934193349852946`. @@ -381,8 +399,8 @@ def extra_repr(self): class GLU(Module): r"""Applies the gated linear unit function - :math:`{GLU}(a, b)= a \otimes \sigma(b)` where `a` is the first half of - the input vector and `b` is the second half. + :math:`{GLU}(a, b)= a \otimes \sigma(b)` where :math:`a` is the first half + of the input vector and :math:`b` is the second half. Args: dim (int): the dimension on which to split the input. Default: -1 @@ -411,8 +429,7 @@ def extra_repr(self): class Hardshrink(Module): - r"""Applies the hard shrinkage function element-wise - Hardshrink is defined as: + r"""Applies the hard shrinkage function element-wise: .. math:: \text{HardShrink}(x) = @@ -451,14 +468,19 @@ def extra_repr(self): class LeakyReLU(Module): - r"""Applies element-wise, - :math:`\text{LeakyReLU}(x) = \max(0, x) + \text{negative_slope} * \min(0, x)` or + r"""Applies the element-wise function: + + .. math:: + \text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x) + + + or .. math:: \text{LeakyRELU}(x) = \begin{cases} x, & \text{ if } x \geq 0 \\ - \text{negative_slope} \times x, & \text{ otherwise } + \text{negative\_slope} \times x, & \text{ otherwise } \end{cases} Args: @@ -493,7 +515,9 @@ def extra_repr(self): class LogSigmoid(Module): - r"""Applies element-wise :math:`\text{LogSigmoid}(x) = \log\left(\frac{ 1 }{ 1 + \exp(-x)}\right)` + r"""Applies the element-wise function: + + .. math:`\text{LogSigmoid}(x) = \log\left(\frac{ 1 }{ 1 + \exp(-x)}\right)` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional @@ -514,7 +538,10 @@ def forward(self, input): class Softplus(Module): - r"""Applies element-wise :math:`\text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x))` + r"""Applies the element-wise function: + + .. math:: + \text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x)) SoftPlus is a smooth approximation to the ReLU function and can be used to constrain the output of a machine to always be positive. @@ -553,9 +580,7 @@ def extra_repr(self): class Softshrink(Module): - r"""Applies the soft shrinkage function elementwise - - SoftShrinkage function is defined as: + r"""Applies the soft shrinkage function elementwise: .. math:: \text{SoftShrinkage}(x) = @@ -594,8 +619,12 @@ def extra_repr(self): class PReLU(Module): - r"""Applies element-wise the function - :math:`\text{PReLU}(x) = \max(0,x) + a * \min(0,x)` or + r"""Applies the element-wise function: + + .. math:: + \text{PReLU}(x) = \max(0,x) + a * \min(0,x) + + or .. math:: \text{PReLU}(x) = @@ -643,7 +672,10 @@ def extra_repr(self): class Softsign(Module): - r"""Applies element-wise, the function :math:`\text{SoftSign}(x) = \frac{x}{ 1 + |x|}` + r"""Applies the element-wise function: + + .. math:: + \text{SoftSign}(x) = \frac{x}{ 1 + |x|} Shape: - Input: :math:`(N, *)` where `*` means, any number of additional @@ -664,7 +696,10 @@ def forward(self, input): class Tanhshrink(Module): - r"""Applies element-wise, :math:`\text{Tanhshrink}(x) = x - \text{Tanh}(x)` + r"""Applies the element-wise function: + + .. math:: + \text{Tanhshrink}(x) = x - \text{Tanh}(x) Shape: - Input: :math:`(N, *)` where `*` means, any number of additional @@ -689,7 +724,8 @@ class Softmin(Module): rescaling them so that the elements of the n-dimensional output Tensor lie in the range `(0, 1)` and sum to 1 - :math:`\text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)}` + .. math:: + \text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)} Shape: - Input: any shape @@ -723,8 +759,10 @@ class Softmax(Module): rescaling them so that the elements of the n-dimensional output Tensor lie in the range (0,1) and sum to 1 - Softmax is defined as - :math:`\text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}` + Softmax is defined as: + + .. math:: + \text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)} Shape: - Input: any shape @@ -791,10 +829,11 @@ def forward(self, input): class LogSoftmax(Module): - r"""Applies the `Log(Softmax(x))` function to an n-dimensional input Tensor. - The LogSoftmax formulation can be simplified as + r"""Applies the :math:`\log(\text{Softmax}(x))` function to an n-dimensional + input Tensor. The LogSoftmax formulation can be simplified as: - :math:`\text{LogSoftmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right)` + .. math:: + \text{LogSoftmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right) Shape: - Input: any shape diff --git a/torch/nn/modules/conv.py b/torch/nn/modules/conv.py index 5af7d29231acc..03cf297e3c0aa 100644 --- a/torch/nn/modules/conv.py +++ b/torch/nn/modules/conv.py @@ -67,15 +67,13 @@ class Conv1d(_ConvNd): planes. In the simplest case, the output value of the layer with input size - :math:`(N, C_{in}, L)` and output :math:`(N, C_{out}, L_{out})` can be + :math:`(N, C_{\text{in}}, L)` and output :math:`(N, C_{\text{out}}, L_{\text{out}})` can be precisely described as: .. math:: - - \begin{equation*} - \text{out}(N_i, C_{out_j}) = \text{bias}(C_{out_j}) + - \sum_{k = 0}^{C_{in} - 1} \text{weight}(C_{out_j}, k) \star \text{input}(N_i, k) - \end{equation*}, + \op{out}(N_i, C_{\text{out}_j}) = \op{bias}(C_{\text{out}_j}) + + \sum_{k = 0}^{C_{in} - 1} \op{weight}(C_{\text{out}_j}, k) + \star \op{input}(N_i, k) where :math:`\star` is the valid `cross-correlation`_ operator, :math:`N` is a batch size, :math:`C` denotes a number of channels, @@ -101,8 +99,9 @@ class Conv1d(_ConvNd): and producing half the output channels, and both subsequently concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with - its own set of filters (of size - :math:`\left\lfloor \frac{\text{out_channels}}{\text{in_channels}} \right\rfloor`). + its own set of filters, + of size + :math:`\left\lfloor\frac{\text{out\_channels}}{\text{in\_channels}}\right\rfloor` .. note:: @@ -119,7 +118,7 @@ class Conv1d(_ConvNd): In other words, for an input of size :math:`(N, C_{in}, L_{in})`, if you want a depthwise convolution with a depthwise multiplier `K`, then you use the constructor arguments - :math:`(\text{in_channels}=C_{in}, \text{out_channels}=C_{in} * K, ..., \text{groups}=C_{in})` + :math:`(\text{in\_channels}=C_{in}, \text{out\_channels}=C_{in} * K, ..., \text{groups}=C_{in})` Args: in_channels (int): Number of channels in the input image @@ -140,17 +139,17 @@ class Conv1d(_ConvNd): .. math:: L_{out} = \left\lfloor\frac{L_{in} + 2 \times \text{padding} - \text{dilation} - \times (\text{kernel_size} - 1) - 1}{\text{stride}} + 1\right\rfloor + \times (\text{kernel\_size} - 1) - 1}{\text{stride}} + 1\right\rfloor Attributes: weight (Tensor): the learnable weights of the module of shape (out_channels, in_channels, kernel_size). The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \text{kernel_size}}` + :math:`k = \frac{1}{\text{in\_channels} * \text{kernel\_size}}` bias (Tensor): the learnable bias of the module of shape (out_channels). If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \text{kernel_size}}` + :math:`k = \frac{1}{\text{in\_channels} * \text{kernel\_size}}` Examples:: @@ -185,15 +184,13 @@ class Conv2d(_ConvNd): planes. In the simplest case, the output value of the layer with input size - :math:`(N, C_{in}, H, W)` and output :math:`(N, C_{out}, H_{out}, W_{out})` + :math:`(N, C_{\text{in}}, H, W)` and output :math:`(N, C_{\text{out}}, H_{\text{out}}, W_{\text{out}})` can be precisely described as: .. math:: + \op{out}(N_i, C_{\text{out}_j}) = \op{bias}(C_{\text{out}_j}) + + \sum_{k = 0}^{C_{\text{in}} - 1} \op{weight}(C_{\text{out}_j}, k) \star \op{input}(N_i, k) - \begin{equation*} - \text{out}(N_i, C_{out_j}) = \text{bias}(C_{out_j}) + - \sum_{k = 0}^{C_{in} - 1} \text{weight}(C_{out_j}, k) \star \text{input}(N_i, k) - \end{equation*}, where :math:`\star` is the valid 2D `cross-correlation`_ operator, :math:`N` is a batch size, :math:`C` denotes a number of channels, @@ -220,8 +217,8 @@ class Conv2d(_ConvNd): and producing half the output channels, and both subsequently concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with - its own set of filters (of size - :math:`\left\lfloor\frac{\text{out_channels}}{\text{in_channels}}\right\rfloor`). + its own set of filters, of size: + :math:`\left\lfloor\frac{\text{out\_channels}}{\text{in\_channels}}\right\rfloor`. The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`dilation` can either be: @@ -244,7 +241,7 @@ class Conv2d(_ConvNd): In other words, for an input of size :math:`(N, C_{in}, H_{in}, W_{in})`, if you want a depthwise convolution with a depthwise multiplier `K`, then you use the constructor arguments - :math:`(\text{in_channels}=C_{in}, \text{out_channels}=C_{in} * K, ..., \text{groups}=C_{in})` + :math:`(in\_channels=C_{in}, out\_channels=C_{in} * K, ..., groups=C_{in})` Args: in_channels (int): Number of channels in the input image @@ -262,21 +259,21 @@ class Conv2d(_ConvNd): .. math:: H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[0] - \text{dilation}[0] - \times (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + \times (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[1] - \text{dilation}[1] - \times (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + \times (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor Attributes: weight (Tensor): the learnable weights of the module of shape (out_channels, in_channels, kernel_size[0], kernel_size[1]). The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{1}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{1}\text{kernel\_size}[i]}` bias (Tensor): the learnable bias of the module of shape (out_channels). If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{1}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{1}\text{kernel\_size}[i]}` Examples:: @@ -319,11 +316,8 @@ class Conv3d(_ConvNd): and output :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})` can be precisely described as: .. math:: - - \begin{equation*} - \text{out}(N_i, C_{out_j}) = \text{bias}(C_{out_j}) + - \sum_{k = 0}^{C_{in} - 1} \text{weight}(C_{out_j}, k) \star \text{input}(N_i, k) - \end{equation*}, + out(N_i, C_{out_j}) = bias(C_{out_j}) + + \sum_{k = 0}^{C_{in} - 1} weight(C_{out_j}, k) \star input(N_i, k) where :math:`\star` is the valid 3D `cross-correlation`_ operator @@ -345,8 +339,8 @@ class Conv3d(_ConvNd): and producing half the output channels, and both subsequently concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with - its own set of filters (of size - :math:`\left\lfloor\frac{\text{out_channels}}{\text{in_channels}}\right\rfloor`). + its own set of filters, of size + :math:`\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor`. The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`dilation` can either be: @@ -369,7 +363,7 @@ class Conv3d(_ConvNd): In other words, for an input of size :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`, if you want a depthwise convolution with a depthwise multiplier `K`, then you use the constructor arguments - :math:`(\text{in_channels}=C_{in}, \text{out_channels}=C_{in} * K, ..., \text{groups}=C_{in})` + :math:`(in\_channels=C_{in}, out\_channels=C_{in} * K, ..., groups=C_{in})` Args: in_channels (int): Number of channels in the input image @@ -387,24 +381,24 @@ class Conv3d(_ConvNd): .. math:: D_{out} = \left\lfloor\frac{D_{in} + 2 \times \text{padding}[0] - \text{dilation}[0] - \times (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + \times (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[1] - \text{dilation}[1] - \times (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + \times (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[2] - \text{dilation}[2] - \times (\text{kernel_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor + \times (\text{kernel\_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor Attributes: weight (Tensor): the learnable weights of the module of shape (out_channels, in_channels, kernel_size[0], kernel_size[1], kernel_size[2]) The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{2}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{2}\text{kernel\_size}[i]}` bias (Tensor): the learnable bias of the module of shape (out_channels). If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{2}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{2}\text{kernel\_size}[i]}` Examples:: @@ -509,7 +503,7 @@ class ConvTranspose1d(_ConvTransposeMixin, _ConvNd): concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with its own set of filters (of size - :math:`\left\lfloor\frac{\text{out_channels}}{\text{in_channels}}\right\rfloor`). + :math:`\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor`). .. note:: @@ -549,18 +543,18 @@ class ConvTranspose1d(_ConvTransposeMixin, _ConvNd): .. math:: L_{out} = (L_{in} - 1) \times \text{stride} - 2 \times \text{padding} - + \text{kernel_size} + \text{output_padding} + + \text{kernel\_size} + \text{output\_padding} Attributes: weight (Tensor): the learnable weights of the module of shape (in_channels, out_channels, kernel_size[0], kernel_size[1]). The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \text{kernel_size}}` + :math:`k = \frac{1}{\text{in\_channels} * \text{kernel\_size}}` bias (Tensor): the learnable bias of the module of shape (out_channels). If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \text{kernel_size}}` + :math:`k = \frac{1}{\text{in\_channels} * \text{kernel\_size}}` """ def __init__(self, in_channels, out_channels, kernel_size, stride=1, @@ -612,7 +606,7 @@ class ConvTranspose2d(_ConvTransposeMixin, _ConvNd): concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with its own set of filters (of size - :math:`\left\lfloor\frac{\text{out_channels}}{\text{in_channels}}\right\rfloor`). + :math:`\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor`). The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`output_padding` can either be: @@ -659,21 +653,21 @@ class ConvTranspose2d(_ConvTransposeMixin, _ConvNd): .. math:: H_{out} = (H_{in} - 1) \times \text{stride}[0] - 2 \times \text{padding}[0] - + \text{kernel_size}[0] + \text{output_padding}[0] + + \text{kernel\_size}[0] + \text{output\_padding}[0] W_{out} = (W_{in} - 1) \times \text{stride}[1] - 2 \times \text{padding}[1] - + \text{kernel_size}[1] + \text{output_padding}[1] + + \text{kernel\_size}[1] + \text{output\_padding}[1] Attributes: weight (Tensor): the learnable weights of the module of shape (in_channels, out_channels, kernel_size[0], kernel_size[1]) The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{1}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{1}\text{kernel\_size}[i]}` bias (Tensor): the learnable bias of the module of shape (out_channels) If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{1}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{1}\text{kernel\_size}[i]}` Examples:: @@ -752,7 +746,7 @@ class ConvTranspose3d(_ConvTransposeMixin, _ConvNd): concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with its own set of filters (of size - :math:`\left\lfloor\frac{\text{out_channels}}{\text{in_channels}}\right\rfloor`). + :math:`\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor`). The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`output_padding` can either be: @@ -799,24 +793,24 @@ class ConvTranspose3d(_ConvTransposeMixin, _ConvNd): .. math:: D_{out} = (D_{in} - 1) \times \text{stride}[0] - 2 \times \text{padding}[0] - + \text{kernel_size}[0] + \text{output_padding}[0] + + \text{kernel\_size}[0] + \text{output\_padding}[0] H_{out} = (H_{in} - 1) \times \text{stride}[1] - 2 \times \text{padding}[1] - + \text{kernel_size}[1] + \text{output_padding}[1] + + \text{kernel\_size}[1] + \text{output\_padding}[1] W_{out} = (W_{in} - 1) \times \text{stride}[2] - 2 \times \text{padding}[2] - + \text{kernel_size}[2] + \text{output_padding}[2] + + \text{kernel\_size}[2] + \text{output\_padding}[2] Attributes: weight (Tensor): the learnable weights of the module of shape (in_channels, out_channels, kernel_size[0], kernel_size[1], kernel_size[2]) The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{2}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{2}\text{kernel\_size}[i]}` bias (Tensor): the learnable bias of the module of shape (out_channels) If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{2}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{2}\text{kernel\_size}[i]}` Examples:: diff --git a/torch/nn/modules/fold.py b/torch/nn/modules/fold.py index 580528e34a5b5..4465b9b10f0e7 100644 --- a/torch/nn/modules/fold.py +++ b/torch/nn/modules/fold.py @@ -8,19 +8,19 @@ class Fold(Module): tensor. Consider a batched :attr:`input` tensor containing sliding local blocks, - e.g., patches of images, of shape :math:`(N, C \times \prod(\text{kernel_size}), L)`, - where :math:`N` is batch dimension, :math:`C \times \prod(\text{kernel_size})` - is the number of values with in a block (a block has :math:`\prod(\text{kernel_size})` + e.g., patches of images, of shape :math:`(N, C \times \prod(\text{kernel\_size}), L)`, + where :math:`N` is batch dimension, :math:`C \times \prod(\text{kernel\_size})` + is the number of values with in a block (a block has :math:`\prod(\text{kernel\_size})` spatial locations each containing a :math:`C`-channeled vector), and :math:`L` is the total number of blocks. (This is exacly the same specification as the output shape of :class:`~torch.nn.Unfold`.) This operation combines these local blocks into the large :attr:`output` tensor - of shape :math:`(N, C, \text{output_size}[0], \text{output_size}[1], \dots)`. + of shape :math:`(N, C, \text{output\_size}[0], \text{output\_size}[1], \dots)`. Similar to :class:`~torch.nn.Unfold`, the arguments must satisfy .. math:: - L = \prod_d \left\lfloor\frac{\text{output_size}[d] + 2 \times \text{padding}[d] \ - - \text{dilation}[d] \times (\text{kernel_size}[d] - 1) - 1}{\text{stride}[d]} + 1\right\rfloor, + L = \prod_d \left\lfloor\frac{\text{output\_size}[d] + 2 \times \text{padding}[d] \ + - \text{dilation}[d] \times (\text{kernel\_size}[d] - 1) - 1}{\text{stride}[d]} + 1\right\rfloor, where :math:`d` is over all spatial dimensions. @@ -64,8 +64,8 @@ class Fold(Module): supported. Shape: - - Input: :math:`(N, C \times \prod(\text{kernel_size}), L)` - - Output: :math:`(N, C, \text{output_size}[0], \text{output_size}[1], \dots)` as described above + - Input: :math:`(N, C \times \prod(\text{kernel\_size}), L)` + - Output: :math:`(N, C, \text{output\_size}[0], \text{output\_size}[1], \dots)` as described above Examples:: @@ -106,17 +106,17 @@ class Unfold(Module): and :math:`*` represent arbitrary spatial dimensions. This operation flattens each sliding :attr:`kernel_size`-sized block within the spatial dimensions of :attr:`input` into a column (i.e., last dimension) of a 3-D :attr:`output` - tensor of shape :math:`(N, C \times \prod(\text{kernel_size}), L)`, where - :math:`C \times \prod(\text{kernel_size})` is the total number of values - with in each block (a block has :math:`\prod(\text{kernel_size})` spatial + tensor of shape :math:`(N, C \times \prod(\text{kernel\_size}), L)`, where + :math:`C \times \prod(\text{kernel\_size})` is the total number of values + with in each block (a block has :math:`\prod(\text{kernel\_size})` spatial locations each containing a :math:`C`-channeled vector), and :math:`L` is the total number of such blocks: .. math:: - L = \prod_d \left\lfloor\frac{\text{input_spatial_size}[d] + 2 \times \text{padding}[d] \ - - \text{dilation}[d] \times (\text{kernel_size}[d] - 1) - 1}{\text{stride}[d]} + 1\right\rfloor, + L = \prod_d \left\lfloor\frac{\text{input\_spatial\_size}[d] + 2 \times \text{padding}[d] \ + - \text{dilation}[d] \times (\text{kernel\_size}[d] - 1) - 1}{\text{stride}[d]} + 1\right\rfloor, - where :math:`\text{input_spatial_size}` is formed by the spatial dimensions + where :math:`\text{input\_spatial\_size}` is formed by the spatial dimensions of :attr:`input` (:math:`*` above), and :math:`d` is over all spatial dimensions. @@ -158,7 +158,7 @@ class Unfold(Module): Shape: - Input: :math:`(N, C, *)` - - Output: :math:`(N, C \times \prod(\text{kernel_size}), L)` as described above + - Output: :math:`(N, C \times \prod(\text{kernel\_size}), L)` as described above Examples:: diff --git a/torch/nn/modules/linear.py b/torch/nn/modules/linear.py index 429340aabc72e..573c6b20cbf03 100644 --- a/torch/nn/modules/linear.py +++ b/torch/nn/modules/linear.py @@ -26,11 +26,11 @@ class Linear(Module): weight: the learnable weights of the module of shape `(out_features x in_features)`. The values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_features}}` - bias: the learnable bias of the module of shape `(out_features)`. + :math:`k = \frac{1}{\text{in\_features}}` + bias: the learnable bias of the module of shape :math:`(out_features)`. If :attr:`bias` is ``True``, the values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_features}}` + :math:`k = \frac{1}{\text{in\_features}}` Examples:: @@ -79,21 +79,21 @@ class Bilinear(Module): Default: ``True`` Shape: - - Input: :math:`(N, *, \text{in1_features})`, :math:`(N, *, \text{in2_features})` + - Input: :math:`(N, *, \text{in1\_features})`, :math:`(N, *, \text{in2\_features})` where :math:`*` means any number of additional dimensions. All but the last dimension of the inputs should be the same. - - Output: :math:`(N, *, \text{out_features})` where all but the last dimension + - Output: :math:`(N, *, \text{out\_features})` where all but the last dimension are the same shape as the input. Attributes: weight: the learnable weights of the module of shape `(out_features x in1_features x in2_features)`. The values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in1_features}}` + :math:`k = \frac{1}{\text{in1\_features}}` bias: the learnable bias of the module of shape `(out_features)` If :attr:`bias` is ``True``, the values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in1_features}}` + :math:`k = \frac{1}{\text{in1\_features}}` Examples:: diff --git a/torch/nn/modules/loss.py b/torch/nn/modules/loss.py index ec7d60d812515..d65e7a0b4c6d3 100644 --- a/torch/nn/modules/loss.py +++ b/torch/nn/modules/loss.py @@ -36,12 +36,13 @@ class L1Loss(_Loss): where :math:`N` is the batch size. If reduce is ``True``, then: .. math:: - \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \ell(x, y) = + \begin{cases} + \operatorname{mean}(L), & \text{if size\_average} = \text{True;}\\ + \operatorname{sum}(L), & \text{if size\_average} = \text{False.} \end{cases} - `x` and `y` arbitrary shapes with a total of `n` elements each. + `x` and `y` are tensors of arbitrary shapes with a total of `n` elements each. The sum operation still operates over all the elements, and divides by `n`. @@ -113,7 +114,7 @@ class NLLLoss(_WeightedLoss): .. math:: \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad l_n = - w_{y_n} x_{n,y_n}, \quad - w_{c} = \text{weight}[c] \cdot \mathbb{1}\{c \not= \text{ignore_index}\}, + w_{c} = \text{weight}[c] \cdot \mathbb{1}\{c \not= \text{ignore\_index}\}, where :math:`N` is the batch size. If :attr:`reduce` is ``True`` (default), then @@ -121,9 +122,9 @@ class NLLLoss(_WeightedLoss): .. math:: \ell(x, y) = \begin{cases} \sum_{n=1}^N \frac{1}{\sum_{n=1}^N w_{y_n}} l_n, & \text{if}\; - \text{size_average} = \text{True},\\ + \text{size\_average} = \text{True},\\ \sum_{n=1}^N l_n, & \text{if}\; - \text{size_average} = \text{False}. + \text{size\_average} = \text{False}. \end{cases} Can also be used for higher dimension inputs, such as 2D images, by providing @@ -290,15 +291,15 @@ class KLDivLoss(_Loss): .. math:: l(x,y) = L := \{ l_1,\dots,l_N \}, \quad - l_n = y_n \cdot \left( \log y_n - x_n \right), + l_n = y_n \cdot \left( \log y_n - x_n \right) where the index :math:`N` spans all dimensions of ``input`` and :math:`L` has the same shape as ``input``. If :attr:`reduce` is ``True`` (the default), then: .. math:: \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \operatorname{mean}(L), & \text{if}\; \text{size\_average} = \text{True},\\ + \operatorname{sum}(L), & \text{if}\; \text{size\_average} = \text{False}. \end{cases} By default, the losses are averaged for each minibatch over observations @@ -371,9 +372,10 @@ class MSELoss(_Loss): where :math:`N` is the batch size. If reduce is ``True``, then: .. math:: - \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \ell(x, y) = + \begin{cases} + \operatorname{mean}(L), & \text{if}\; \text{size\_average} = \text{True},\\ + \operatorname{sum}(L), & \text{if}\; \text{size\_average} = \text{False}. \end{cases} The sum operation still operates over all the elements, and divides by `n`. @@ -435,8 +437,8 @@ class BCELoss(_WeightedLoss): .. math:: \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \operatorname{mean}(L), & \text{if}\; \text{size\_average} = \text{True},\\ + \operatorname{sum}(L), & \text{if}\; \text{size\_average} = \text{False}. \end{cases} This is used for measuring the error of a reconstruction in for example @@ -503,8 +505,8 @@ class BCEWithLogitsLoss(_Loss): .. math:: \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \operatorname{mean}(L), & \text{if size\_average} = \text{True},\\ + \operatorname{sum}(L), & \text{if size\_average} = \text{False}. \end{cases} This is used for measuring the error of a reconstruction in for example @@ -592,8 +594,8 @@ class HingeEmbeddingLoss(_Loss): .. math:: \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \operatorname{mean}(L), & \text{if size\_average} = \text{True},\\ + \operatorname{sum}(L), & \text{if size\_average} = \text{False}. \end{cases} where :math:`L = \{l_1,\dots,l_N\}^\top`. @@ -639,8 +641,10 @@ class MultiLabelMarginLoss(_Loss): .. math:: \text{loss}(x, y) = \sum_{ij}\frac{\max(0, 1 - (x[y[j]] - x[i]))}{\text{x.size}(0)} - where `i == 0` to `x.size(0)`, `j == 0` to `y.size(0)`, - :math:`y[j] \geq 0`, and :math:`i \neq y[j]` for all `i` and `j`. + where :math:`i == 0` to :math:`x.size(0)`, \ + :math:`j == 0` to :math:`y.size(0)`, \ + :math:`y[j] \geq 0`, \ + and :math:`i \neq y[j]` for all :math:`i` and :math:`j`. `y` and `x` must have the same size. @@ -1072,7 +1076,11 @@ class TripletMarginLoss(_Loss): .. math:: L(a, p, n) = \max \{d(a_i, p_i) - d(a_i, n_i) + {\rm margin}, 0\} - where :math:`d(x_i, y_i) = \left\lVert {\bf x}_i - {\bf y}_i \right\rVert_p`. + + where + + .. math:: + d(x_i, y_i) = \left\lVert {\bf x}_i - {\bf y}_i \right\rVert_p Args: margin (float, optional): Default: `1`. diff --git a/torch/nn/modules/normalization.py b/torch/nn/modules/normalization.py index a40cb314a6b8f..b3bf05f9d5307 100644 --- a/torch/nn/modules/normalization.py +++ b/torch/nn/modules/normalization.py @@ -95,8 +95,8 @@ class LayerNorm(Module): of size .. math:: - [* \times \text{normalized_shape}[0] \times \text{normalized_shape}[1] - \times \ldots \times \text{normalized_shape}[-1]] + [* \times \text{normalized\_shape}[0] \times \text{normalized\_shape}[1] + \times \ldots \times \text{normalized\_shape}[-1]] If a single integer is used, it is treated as a singleton list, and this module will normalize over the last dimension which is expected to be of that specific size. diff --git a/torch/nn/modules/padding.py b/torch/nn/modules/padding.py index 2ad9a5624a6a3..6358e8e0dae20 100644 --- a/torch/nn/modules/padding.py +++ b/torch/nn/modules/padding.py @@ -26,39 +26,39 @@ class ConstantPad1d(_ConstantPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in both boundaries. If a 2-`tuple`, uses (`paddingLeft`, `paddingRight`) + padding in both boundaries. If a 2-`tuple`, uses + (:math:`\text{padding\_left}`, :math:`\text{padding\_right}`) Shape: - Input: :math:`(N, C, W_{in})` - Output: :math:`(N, C, W_{out})` where - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ConstantPad1d(2, 3.5) >>> input = torch.randn(1, 2, 4) >>> input - - (0 ,.,.) = - 0.1875 0.5046 -1.0074 2.0005 - -0.3540 -1.8645 1.1530 0.0632 - [torch.FloatTensor of size (1,2,4)] - + tensor([[[-1.0491, -0.7152, -0.0749, 0.8530], + [-1.3287, 1.8966, 0.1466, -0.2771]]]) >>> m(input) - - (0 ,.,.) = - 3.5000 3.5000 0.1875 0.5046 -1.0074 2.0005 3.5000 3.5000 - 3.5000 3.5000 -0.3540 -1.8645 1.1530 0.0632 3.5000 3.5000 - [torch.FloatTensor of size (1,2,8)] - - >>> # using different paddings + tensor([[[ 3.5000, 3.5000, -1.0491, -0.7152, -0.0749, 0.8530, 3.5000, + 3.5000], + [ 3.5000, 3.5000, -1.3287, 1.8966, 0.1466, -0.2771, 3.5000, + 3.5000]]]) + >>> m = nn.ConstantPad1d(2, 3.5) + >>> input = torch.randn(1, 2, 3) + >>> input + tensor([[[ 1.6616, 1.4523, -1.1255], + [-3.6372, 0.1182, -1.8652]]]) + >>> m(input) + tensor([[[ 3.5000, 3.5000, 1.6616, 1.4523, -1.1255, 3.5000, 3.5000], + [ 3.5000, 3.5000, -3.6372, 0.1182, -1.8652, 3.5000, 3.5000]]]) + >>> # using different paddings for different sides >>> m = nn.ConstantPad1d((3, 1), 3.5) >>> m(input) - - (0 ,.,.) = - 3.5000 3.5000 3.5000 0.1875 0.5046 -1.0074 2.0005 3.5000 - 3.5000 3.5000 3.5000 -0.3540 -1.8645 1.1530 0.0632 3.5000 - [torch.FloatTensor of size (1,2,8)] + tensor([[[ 3.5000, 3.5000, 3.5000, 1.6616, 1.4523, -1.1255, 3.5000], + [ 3.5000, 3.5000, 3.5000, -3.6372, 0.1182, -1.8652, 3.5000]]]) """ @@ -74,48 +74,44 @@ class ConstantPad2d(_ConstantPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 4-`tuple`, uses (`paddingLeft`, `paddingRight`, - `paddingTop`, `paddingBottom`) + padding in all boundaries. If a 4-`tuple`, uses (:math:`\text{padding\_left}`, + :math:`\text{padding\_right}`, :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`) Shape: - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ConstantPad2d(2, 3.5) >>> input = torch.randn(1, 2, 2) >>> input - - (0 ,.,.) = - -0.2295 -0.9774 - -0.3335 -1.4178 - [torch.FloatTensor of size (1,2,2)] - + tensor([[[ 1.6585, 0.4320], + [-0.8701, -0.4649]]]) >>> m(input) - - (0 ,.,.) = - 3.5000 3.5000 3.5000 3.5000 3.5000 3.5000 - 3.5000 3.5000 3.5000 3.5000 3.5000 3.5000 - 3.5000 3.5000 -0.2295 -0.9774 3.5000 3.5000 - 3.5000 3.5000 -0.3335 -1.4178 3.5000 3.5000 - 3.5000 3.5000 3.5000 3.5000 3.5000 3.5000 - 3.5000 3.5000 3.5000 3.5000 3.5000 3.5000 - [torch.FloatTensor of size (1,6,6)] - - >>> # using different paddings + tensor([[[ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 1.6585, 0.4320, 3.5000, 3.5000], + [ 3.5000, 3.5000, -0.8701, -0.4649, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000]]]) + >>> m(input) + tensor([[[ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 1.6585, 0.4320, 3.5000, 3.5000], + [ 3.5000, 3.5000, -0.8701, -0.4649, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000]]]) + >>> # using different paddings for different sides >>> m = nn.ConstantPad2d((3, 0, 2, 1), 3.5) >>> m(input) - - (0 ,.,.) = - 3.5000 3.5000 3.5000 3.5000 3.5000 - 3.5000 3.5000 3.5000 3.5000 3.5000 - 3.5000 3.5000 3.5000 -0.2295 -0.9774 - 3.5000 3.5000 3.5000 -0.3335 -1.4178 - 3.5000 3.5000 3.5000 3.5000 3.5000 - [torch.FloatTensor of size (1,5,5)] + tensor([[[ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 1.6585, 0.4320], + [ 3.5000, 3.5000, 3.5000, -0.8701, -0.4649], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000]]]) """ @@ -132,21 +128,23 @@ class ConstantPad3d(_ConstantPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same padding in all boundaries. If a 6-`tuple`, uses - (`paddingLeft`, `paddingRight`, `paddingTop`, `paddingBottom`, `paddingFront`, `paddingBack`) + (:math:`\text{padding\_left}`, :math:`\text{padding\_right}`, + :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`, + :math:`\text{padding\_front}`, :math:`\text{padding\_back}`) Shape: - Input: :math:`(N, C, D_{in}, H_{in}, W_{in})` - Output: :math:`(N, C, D_{out}, H_{out}, W_{out})` where - :math:`D_{out} = D_{in} + \textit{paddingFront} + \textit{paddingBack}` - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`D_{out} = D_{in} + \text{padding\_front} + \text{padding\_back}` + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ConstantPad3d(3, 3.5) >>> input = torch.randn(16, 3, 10, 20, 30) >>> output = m(input) - >>> # using different paddings + >>> # using different paddings for different sides >>> m = nn.ConstantPad3d((3, 3, 6, 6, 0, 1), 3.5) >>> output = m(input) @@ -173,39 +171,32 @@ class ReflectionPad1d(_ReflectionPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 2-`tuple`, uses (`paddingLeft`, `paddingRight`) + padding in all boundaries. If a 2-`tuple`, uses + (:math:`\text{padding\_left}`, :math:`\text{padding\_right}`) Shape: - Input: :math:`(N, C, W_{in})` - Output: :math:`(N, C, W_{out})` where - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ReflectionPad1d(2) - >>> input = torch.arange(8).reshape(1, 2, 4) + >>> input = torch.arange(8, dtype=torch.float).reshape(1, 2, 4) >>> input - - (0 ,.,.) = - 0 1 2 3 - 4 5 6 7 - [torch.FloatTensor of size (1,2,4)] - + tensor([[[0., 1., 2., 3.], + [4., 5., 6., 7.]]]) >>> m(input) - - (0 ,.,.) = - 2 1 0 1 2 3 2 1 - 6 5 4 5 6 7 6 5 - [torch.FloatTensor of size (1,2,8)] - - >>> # using different paddings + tensor([[[2., 1., 0., 1., 2., 3., 2., 1.], + [6., 5., 4., 5., 6., 7., 6., 5.]]]) + >>> m(input) + tensor([[[2., 1., 0., 1., 2., 3., 2., 1.], + [6., 5., 4., 5., 6., 7., 6., 5.]]]) + >>> # using different paddings for different sides >>> m = nn.ReflectionPad1d((3, 1)) >>> m(input) - - (0 ,.,.) = - 3 2 1 0 1 2 3 2 - 7 6 5 4 5 6 7 6 - [torch.FloatTensor of size (1,2,8)] + tensor([[[3., 2., 1., 0., 1., 2., 3., 2.], + [7., 6., 5., 4., 5., 6., 7., 6.]]]) """ @@ -221,50 +212,40 @@ class ReflectionPad2d(_ReflectionPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 4-`tuple`, uses (`paddingLeft`, `paddingRight`, - `paddingTop`, `paddingBottom`) + padding in all boundaries. If a 4-`tuple`, uses (:math:`\text{padding\_left}`, + :math:`\text{padding\_right}`, :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`) Shape: - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ReflectionPad2d(2) - >>> input = torch.arange(9).reshape(1, 1, 3, 3) + >>> input = torch.arange(9, dtype=torch.float).reshape(1, 1, 3, 3) >>> input - - (0 ,0 ,.,.) = - 0 1 2 - 3 4 5 - 6 7 8 - [torch.FloatTensor of size (1,1,3,3)] - + tensor([[[[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]]]]) >>> m(input) - - (0 ,0 ,.,.) = - 8 7 6 7 8 7 6 - 5 4 3 4 5 4 3 - 2 1 0 1 2 1 0 - 5 4 3 4 5 4 3 - 8 7 6 7 8 7 6 - 5 4 3 4 5 4 3 - 2 1 0 1 2 1 0 - [torch.FloatTensor of size (1,1,7,7)] - - >>> # using different paddings + tensor([[[[8., 7., 6., 7., 8., 7., 6.], + [5., 4., 3., 4., 5., 4., 3.], + [2., 1., 0., 1., 2., 1., 0.], + [5., 4., 3., 4., 5., 4., 3.], + [8., 7., 6., 7., 8., 7., 6.], + [5., 4., 3., 4., 5., 4., 3.], + [2., 1., 0., 1., 2., 1., 0.]]]]) + >>> # using different paddings for different sides >>> m = nn.ReflectionPad2d((1, 1, 2, 0)) >>> m(input) - - (0 ,0 ,.,.) = - 7 6 7 8 7 - 4 3 4 5 4 - 1 0 1 2 1 - 4 3 4 5 4 - 7 6 7 8 7 - [torch.FloatTensor of size (1,1,5,5)] + tensor([[[[7., 6., 7., 8., 7.], + [4., 3., 4., 5., 4.], + [1., 0., 1., 2., 1.], + [4., 3., 4., 5., 4.], + [7., 6., 7., 8., 7.]]]]) """ @@ -289,39 +270,29 @@ class ReplicationPad1d(_ReplicationPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 2-`tuple`, uses (`paddingLeft`, `paddingRight`) + padding in all boundaries. If a 2-`tuple`, uses + (:math:`\text{padding\_left}`, :math:`\text{padding\_right}`) Shape: - Input: :math:`(N, C, W_{in})` - Output: :math:`(N, C, W_{out})` where - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ReplicationPad1d(2) - >>> input = torch.arange(8).reshape(1, 2, 4) + >>> input = torch.arange(8, dtype=torch.float).reshape(1, 2, 4) >>> input - - (0 ,.,.) = - 0 1 2 3 - 4 5 6 7 - [torch.FloatTensor of size (1,2,4)] - + tensor([[[0., 1., 2., 3.], + [4., 5., 6., 7.]]]) >>> m(input) - - (0 ,.,.) = - 0 0 0 1 2 3 3 3 - 4 4 4 5 6 7 7 7 - [torch.FloatTensor of size (1,2,8)] - - >>> # using different paddings + tensor([[[0., 0., 0., 1., 2., 3., 3., 3.], + [4., 4., 4., 5., 6., 7., 7., 7.]]]) + >>> # using different paddings for different sides >>> m = nn.ReplicationPad1d((3, 1)) >>> m(input) - - (0 ,.,.) = - 0 0 0 0 1 2 3 3 - 4 4 4 4 5 6 7 7 - [torch.FloatTensor of size (1,2,8)] + tensor([[[0., 0., 0., 0., 1., 2., 3., 3.], + [4., 4., 4., 4., 5., 6., 7., 7.]]]) """ @@ -337,50 +308,39 @@ class ReplicationPad2d(_ReplicationPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 4-`tuple`, uses (`paddingLeft`, `paddingRight`, - `paddingTop`, `paddingBottom`) + padding in all boundaries. If a 4-`tuple`, uses (:math:`\text{padding\_left}`, + :math:`\text{padding\_right}`, :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`) Shape: - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ReplicationPad2d(2) - >>> input = torch.arange(9).reshape(1, 1, 3, 3) + >>> input = torch.arange(9, dtype=torch.float).reshape(1, 1, 3, 3) >>> input - - (0 ,0 ,.,.) = - 0 1 2 - 3 4 5 - 6 7 8 - [torch.FloatTensor of size (1,1,3,3)] - + tensor([[[[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]]]]) >>> m(input) - - (0 ,0 ,.,.) = - 0 0 0 1 2 2 2 - 0 0 0 1 2 2 2 - 0 0 0 1 2 2 2 - 3 3 3 4 5 5 5 - 6 6 6 7 8 8 8 - 6 6 6 7 8 8 8 - 6 6 6 7 8 8 8 - [torch.FloatTensor of size (1,1,7,7)] - - >>> # using different paddings + tensor([[[[0., 0., 0., 1., 2., 2., 2.], + [0., 0., 0., 1., 2., 2., 2.], + [0., 0., 0., 1., 2., 2., 2.], + [3., 3., 3., 4., 5., 5., 5.], + [6., 6., 6., 7., 8., 8., 8.], + [6., 6., 6., 7., 8., 8., 8.], + [6., 6., 6., 7., 8., 8., 8.]]]]) + >>> # using different paddings for different sides >>> m = nn.ReplicationPad2d((1, 1, 2, 0)) >>> m(input) - - (0 ,0 ,.,.) = - 0 0 1 2 2 - 0 0 1 2 2 - 0 0 1 2 2 - 3 3 4 5 5 - 6 6 7 8 8 - [torch.FloatTensor of size (1,1,5,5)] + tensor([[[[0., 0., 1., 2., 2.], + [0., 0., 1., 2., 2.], + [0., 0., 1., 2., 2.], + [3., 3., 4., 5., 5.], + [6., 6., 7., 8., 8.]]]]) """ @@ -396,22 +356,24 @@ class ReplicationPad3d(_ReplicationPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 6-`tuple`, uses (`paddingLeft`, `paddingRight`, - `paddingTop`, `paddingBottom`, `paddingFront`, `paddingBack`) + padding in all boundaries. If a 6-`tuple`, uses + (:math:`\text{padding\_left}`, :math:`\text{padding\_right}`, + :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`, + :math:`\text{padding\_front}`, :math:`\text{padding\_back}`) Shape: - Input: :math:`(N, C, D_{in}, H_{in}, W_{in})` - Output: :math:`(N, C, D_{out}, H_{out}, W_{out})` where - :math:`D_{out} = D_{in} + \textit{paddingFront} + \textit{paddingBack}` - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`D_{out} = D_{in} + \text{padding\_front} + \text{padding\_back}` + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ReplicationPad3d(3) >>> input = torch.randn(16, 3, 8, 320, 480) >>> output = m(input) - >>> # using different paddings + >>> # using different paddings for different sides >>> m = nn.ReplicationPad3d((3, 3, 6, 6, 1, 1)) >>> output = m(input) @@ -429,50 +391,39 @@ class ZeroPad2d(ConstantPad2d): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 4-`tuple`, uses (`paddingLeft`, `paddingRight`, - `paddingTop`, `paddingBottom`) + padding in all boundaries. If a 4-`tuple`, uses (:math:`\text{padding\_left}`, + :math:`\text{padding\_right}`, :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`) Shape: - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ZeroPad2d(2) >>> input = torch.randn(1, 1, 3, 3) >>> input - - (0 ,0 ,.,.) = - 1.4418 -1.9812 -0.3815 - -0.3828 -0.6833 -0.2376 - 0.1433 0.0211 0.4311 - [torch.FloatTensor of size (1,1,3,3)] - + tensor([[[[-0.1678, -0.4418, 1.9466], + [ 0.9604, -0.4219, -0.5241], + [-0.9162, -0.5436, -0.6446]]]]) >>> m(input) - - (0 ,0 ,.,.) = - 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 - 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 - 0.0000 0.0000 1.4418 -1.9812 -0.3815 0.0000 0.0000 - 0.0000 0.0000 -0.3828 -0.6833 -0.2376 0.0000 0.0000 - 0.0000 0.0000 0.1433 0.0211 0.4311 0.0000 0.0000 - 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 - 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 - [torch.FloatTensor of size (1,1,7,7)] - - >>> # using different paddings + tensor([[[[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, 0.0000, -0.1678, -0.4418, 1.9466, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.9604, -0.4219, -0.5241, 0.0000, 0.0000], + [ 0.0000, 0.0000, -0.9162, -0.5436, -0.6446, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000]]]]) + >>> # using different paddings for different sides >>> m = nn.ZeroPad2d((1, 1, 2, 0)) >>> m(input) - - (0 ,0 ,.,.) = - 0.0000 0.0000 0.0000 0.0000 0.0000 - 0.0000 0.0000 0.0000 0.0000 0.0000 - 0.0000 1.4418 -1.9812 -0.3815 0.0000 - 0.0000 -0.3828 -0.6833 -0.2376 0.0000 - 0.0000 0.1433 0.0211 0.4311 0.0000 - [torch.FloatTensor of size (1,1,5,5)] + tensor([[[[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, -0.1678, -0.4418, 1.9466, 0.0000], + [ 0.0000, 0.9604, -0.4219, -0.5241, 0.0000], + [ 0.0000, -0.9162, -0.5436, -0.6446, 0.0000]]]]) """ diff --git a/torch/nn/modules/pixelshuffle.py b/torch/nn/modules/pixelshuffle.py index 26c325f72f631..e68946d7c7fe9 100644 --- a/torch/nn/modules/pixelshuffle.py +++ b/torch/nn/modules/pixelshuffle.py @@ -17,8 +17,8 @@ class PixelShuffle(Module): upscale_factor (int): factor to increase spatial resolution by Shape: - - Input: :math:`(N, C * \text{upscale_factor}^2, H, W)` - - Output: :math:`(N, C, H * \text{upscale_factor}, W * \text{upscale_factor})` + - Input: :math:`(N, C * \text{upscale\_factor}^2, H, W)` + - Output: :math:`(N, C, H * \text{upscale\_factor}, W * \text{upscale\_factor})` Examples:: diff --git a/torch/nn/modules/pooling.py b/torch/nn/modules/pooling.py index 75ab843e5413b..7ca95d21230ca 100644 --- a/torch/nn/modules/pooling.py +++ b/torch/nn/modules/pooling.py @@ -30,11 +30,8 @@ class MaxPool1d(_MaxPoolNd): and output :math:`(N, C, L_{out})` can be precisely described as: .. math:: - - \begin{equation*} - \text{out}(N_i, C_j, k) = \max_{m=0, \ldots, \text{kernel_size}-1} - \text{input}(N_i, C_j, \text{stride} * k + m) - \end{equation*} + out(N_i, C_j, k) = \max_{m=0, \ldots, kernel\_size-1} + input(N_i, C_j, stride * k + m) If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides for :attr:`padding` number of points. :attr:`dilation` controls the spacing between the kernel points. @@ -55,7 +52,7 @@ class MaxPool1d(_MaxPoolNd): .. math:: L_{out} = \left\lfloor \frac{L_{in} + 2 * \text{padding} - \text{dilation} - * (\text{kernel_size} - 1) - 1}{\text{stride}} + 1\right\rfloor + * (\text{kernel\_size} - 1) - 1}{\text{stride}} + 1\right\rfloor Examples:: @@ -88,10 +85,8 @@ class MaxPool2d(_MaxPoolNd): .. math:: - \begin{equation*} - \text{out}(N_i, C_j, h, w) = \max_{m=0, \ldots, kH-1} \max_{n=0, \ldots, kW-1} - \text{input}(N_i, C_j, \text{stride}[0] * h + m, \text{stride}[1] * w + n) - \end{equation*} + out(N_i, C_j, h, w) = \max_{m=0, \ldots, kH-1} \max_{n=0, \ldots, kW-1} + \text{input}(N_i, C_j, \text{stride[0]} * h + m, \text{stride[1]} * w + n) If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides for :attr:`padding` number of points. :attr:`dilation` controls the spacing between the kernel points. @@ -117,11 +112,12 @@ class MaxPool2d(_MaxPoolNd): - Output: :math:`(N, C, H_{out}, W_{out})` where .. math:: - H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding}[0] - \text{dilation}[0] - * (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding[0]} - \text{dilation[0]} + * (\text{kernel\_size[0]} - 1) - 1}{\text{stride[0]}} + 1\right\rfloor - W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding}[1] - \text{dilation}[1] - * (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + .. math:: + W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding[1]} - \text{dilation[1]} + * (\text{kernel\_size[1]} - 1) - 1}{\text{stride[1]}} + 1\right\rfloor Examples:: @@ -144,18 +140,21 @@ def forward(self, input): class MaxPool3d(_MaxPoolNd): r"""Applies a 3D max pooling over an input signal composed of several input - planes. + planes. This is not a test In the simplest case, the output value of the layer with input size :math:`(N, C, D, H, W)`, output :math:`(N, C, D_{out}, H_{out}, W_{out})` and :attr:`kernel_size` :math:`(kD, kH, kW)` can be precisely described as: .. math:: - - \begin{align*} - \text{out}(N_i, C_j, d, h, w) &= \max_{k=0, \ldots, kD-1} \max_{m=0, \ldots, kH-1} \max_{n=0, \ldots, kW-1} - \text{input}(N_i, C_j, \text{stride}[0] * k + d,\\ &\text{stride}[1] * h + m, \text{stride}[2] * w + n) - \end{align*} + out(N_i, C_j, d, h, w) = + \begin{gathered} + \max_{k=0, \ldots, kD-1} + \max_{m=0, \ldots, kH-1} + \max_{n=0, \ldots, kW-1} \\ + \text{input}(N_i, C_j, \text{stride[0]} * + k + d, \text{stride[1]} * h + m, \text{stride[2]} * w + n) + \end{gathered} If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides for :attr:`padding` number of points. :attr:`dilation` controls the spacing between the kernel points. @@ -182,13 +181,15 @@ class MaxPool3d(_MaxPoolNd): .. math:: D_{out} = \left\lfloor\frac{D_{in} + 2 * \text{padding}[0] - \text{dilation}[0] * - (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + .. math:: H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding}[1] - \text{dilation}[1] * - (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + .. math:: W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding}[2] - \text{dilation}[2] * - (\text{kernel_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor + (\text{kernel\_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor Examples:: @@ -248,7 +249,7 @@ class MaxUnpool1d(_MaxUnpoolNd): - Output: :math:`(N, C, H_{out})` where .. math:: - H_{out} = (H_{in} - 1) * \text{stride}[0] - 2 * \text{padding}[0] + \text{kernel_size}[0] + H_{out} = (H_{in} - 1) * \text{stride}[0] - 2 * \text{padding}[0] + \text{kernel\_size}[0] or as given by :attr:`output_size` in the call operator @@ -313,9 +314,10 @@ class MaxUnpool2d(_MaxUnpoolNd): - Output: :math:`(N, C, H_{out}, W_{out})` where .. math:: - H_{out} = (H_{in} - 1) * \text{stride}[0] - 2 * \text{padding}[0] + \text{kernel_size}[0] + H_{out} = (H_{in} - 1) * \text{stride[0]} - 2 * \text{padding[0]} + \text{kernel\_size[0]} - W_{out} = (W_{in} - 1) * \text{stride}[1] - 2 * \text{padding}[1] + \text{kernel_size}[1] + .. math:: + W_{out} = (W_{in} - 1) * \text{stride[1]} - 2 * \text{padding[1]} + \text{kernel\_size[1]} or as given by :attr:`output_size` in the call operator @@ -384,11 +386,13 @@ class MaxUnpool3d(_MaxUnpoolNd): - Output: :math:`(N, C, D_{out}, H_{out}, W_{out})` where .. math:: - D_{out} = (D_{in} - 1) * \text{stride}[0] - 2 * \text{padding}[0] + \text{kernel_size}[0] + D_{out} = (D_{in} - 1) * \text{stride[0]} - 2 * \text{padding[0]} + \text{kernel\_size[0]} - H_{out} = (H_{in} - 1) * \text{stride}[1] - 2 * \text{padding}[1] + \text{kernel_size}[1] + .. math:: + H_{out} = (H_{in} - 1) * \text{stride[1]} - 2 * \text{padding[1]} + \text{kernel\_size[1]} - W_{out} = (W_{in} - 1) * \text{stride}[2] - 2 * \text{padding}[2] + \text{kernel_size}[2] + .. math:: + W_{out} = (W_{in} - 1) * \text{stride[2]} - 2 * \text{padding[2]} + \text{kernel\_size[2]} or as given by :attr:`output_size` in the call operator @@ -432,10 +436,8 @@ class AvgPool1d(_AvgPoolNd): .. math:: - \begin{equation*} \text{out}(N_i, C_j, l) = \frac{1}{k} \sum_{m=0}^{k} \text{input}(N_i, C_j, \text{stride} * l + m) - \end{equation*} If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides for :attr:`padding` number of points. @@ -456,7 +458,7 @@ class AvgPool1d(_AvgPoolNd): .. math:: L_{out} = \left\lfloor \frac{L_{in} + - 2 * \text{padding} - \text{kernel_size}}{\text{stride}} + 1\right\rfloor + 2 * \text{padding} - \text{kernel\_size}}{\text{stride}} + 1\right\rfloor Examples:: @@ -491,10 +493,8 @@ class AvgPool2d(_AvgPoolNd): .. math:: - \begin{equation*} - \text{out}(N_i, C_j, h, w) = \frac{1}{kH * kW} \sum_{m=0}^{kH-1} \sum_{n=0}^{kW-1} - \text{input}(N_i, C_j, \text{stride}[0] * h + m, \text{stride}[1] * w + n) - \end{equation*} + out(N_i, C_j, h, w) = \frac{1}{kH * kW} \sum_{m=0}^{kH-1} \sum_{n=0}^{kW-1} + input(N_i, C_j, stride[0] * h + m, stride[1] * w + n) If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides for :attr:`padding` number of points. @@ -518,10 +518,11 @@ class AvgPool2d(_AvgPoolNd): .. math:: H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding}[0] - - \text{kernel_size}[0]}{\text{stride}[0]} + 1\right\rfloor + \text{kernel\_size}[0]}{\text{stride}[0]} + 1\right\rfloor + .. math:: W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding}[1] - - \text{kernel_size}[1]}{\text{stride}[1]} + 1\right\rfloor + \text{kernel\_size}[1]}{\text{stride}[1]} + 1\right\rfloor Examples:: @@ -557,12 +558,10 @@ class AvgPool3d(_AvgPoolNd): .. math:: - \begin{equation*} \text{out}(N_i, C_j, d, h, w) = \sum_{k=0}^{kD-1} \sum_{m=0}^{kH-1} \sum_{n=0}^{kW-1} \frac{\text{input}(N_i, C_j, \text{stride}[0] * d + k, \text{stride}[1] * h + m, \text{stride}[2] * w + n)} {kD * kH * kW} - \end{equation*} If :attr:`padding` is non-zero, then the input is implicitly zero-padded on all three sides for :attr:`padding` number of points. @@ -586,13 +585,15 @@ class AvgPool3d(_AvgPoolNd): .. math:: D_{out} = \left\lfloor\frac{D_{in} + 2 * \text{padding}[0] - - \text{kernel_size}[0]}{\text{stride}[0]} + 1\right\rfloor + \text{kernel\_size}[0]}{\text{stride}[0]} + 1\right\rfloor + .. math:: H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding}[1] - - \text{kernel_size}[1]}{\text{stride}[1]} + 1\right\rfloor + \text{kernel\_size}[1]}{\text{stride}[1]} + 1\right\rfloor + .. math:: W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding}[2] - - \text{kernel_size}[2]}{\text{stride}[2]} + 1\right\rfloor + \text{kernel\_size}[2]}{\text{stride}[2]} + 1\right\rfloor Examples:: @@ -721,7 +722,7 @@ class LPPool1d(_LPPoolNd): .. math:: L_{out} = \left\lfloor\frac{L_{in} + - 2 * \text{padding} - \text{kernel_size}}{\text{stride}} + 1\right\rfloor + 2 * \text{padding} - \text{kernel\_size}}{\text{stride}} + 1\right\rfloor Examples:: >>> # power-2 pool of window of length 3, with stride 2. @@ -767,10 +768,11 @@ class LPPool2d(_LPPoolNd): .. math:: H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding}[0] - \text{dilation}[0] * - (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + .. math:: W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding}[1] - \text{dilation}[1] * - (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor Examples:: diff --git a/torch/nn/modules/rnn.py b/torch/nn/modules/rnn.py index 9f726d12e7a72..45d5a216ff332 100644 --- a/torch/nn/modules/rnn.py +++ b/torch/nn/modules/rnn.py @@ -235,7 +235,7 @@ def all_weights(self): class RNN(RNNBase): - r"""Applies a multi-layer Elman RNN with `tanh` or `ReLU` non-linearity to an + r"""Applies a multi-layer Elman RNN with :math:`tanh` or :math:`ReLU` non-linearity to an input sequence. @@ -243,8 +243,7 @@ class RNN(RNNBase): function: .. math:: - - h_t = \tanh(w_{ih} x_t + b_{ih} + w_{hh} h_{(t-1)} + b_{hh}) + h_t = \text{tanh}(w_{ih} x_t + b_{ih} + w_{hh} h_{(t-1)} + b_{hh}) where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the input at time `t`, and :math:`h_{(t-1)}` is the hidden state of the @@ -307,7 +306,7 @@ class RNN(RNNBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: @@ -342,15 +341,14 @@ class LSTM(RNNBase): function: .. math:: - - \begin{array}{ll} + \begin{array}{ll} \\ i_t = \sigma(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\ f_t = \sigma(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\ g_t = \tanh(W_{ig} x_t + b_{ig} + W_{hg} h_{(t-1)} + b_{hg}) \\ o_t = \sigma(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\ c_t = f_t c_{(t-1)} + i_t g_t \\ - h_t = o_t \tanh(c_t) - \end{array} + h_t = o_t \tanh(c_t) \\ + \end{array} where :math:`h_t` is the hidden state at time `t`, :math:`c_t` is the cell state at time `t`, :math:`x_t` is the input at time `t`, :math:`h_{(t-1)}` @@ -419,7 +417,7 @@ class LSTM(RNNBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: @@ -442,13 +440,12 @@ class GRU(RNNBase): function: .. math:: - - \begin{array}{ll} + \begin{array}{ll} r_t = \sigma(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\ z_t = \sigma(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\ n_t = \tanh(W_{in} x_t + b_{in} + r_t (W_{hn} h_{(t-1)}+ b_{hn})) \\ - h_t = (1 - z_t) n_t + z_t h_{(t-1)} \\ - \end{array} + h_t = (1 - z_t) n_t + z_t h_{(t-1)} + \end{array} where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the input at time `t`, :math:`h_{(t-1)}` is the hidden state of the previous layer @@ -509,7 +506,7 @@ class GRU(RNNBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: @@ -587,7 +584,7 @@ class RNNCell(RNNCellBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: @@ -688,7 +685,7 @@ class LSTMCell(RNNCellBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: @@ -776,7 +773,7 @@ class GRUCell(RNNCellBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: diff --git a/torch/nn/modules/upsampling.py b/torch/nn/modules/upsampling.py index 5e0e6671374a9..e416326229a14 100644 --- a/torch/nn/modules/upsampling.py +++ b/torch/nn/modules/upsampling.py @@ -33,12 +33,14 @@ class Upsample(Module): - Output: :math:`(N, C, W_{out})`, :math:`(N, C, H_{out}, W_{out})` or :math:`(N, C, D_{out}, H_{out}, W_{out})`, where - .. math:: - D_{out} = \left\lfloor D_{in} \times \text{scale_factor} \right\rfloor \text{ or size}[-3] + .. math:: + D_{out} = \left\lfloor D_{in} \times \text{scale\_factor} \right\rfloor \text{ or size}[-3] - H_{out} = \left\lfloor H_{in} \times \text{scale_factor} \right\rfloor \text{ or size}[-2] + .. math:: + H_{out} = \left\lfloor H_{in} \times \text{scale\_factor} \right\rfloor \text{ or size}[-2] - W_{out} = \left\lfloor W_{in} \times \text{scale_factor} \right\rfloor \text{ or size}[-1] + .. math:: + W_{out} = \left\lfloor W_{in} \times \text{scale\_factor} \right\rfloor \text{ or size}[-1] .. warning:: With ``align_corners = True``, the linearly interpolating modes @@ -151,10 +153,11 @@ class UpsamplingNearest2d(Upsample): - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - .. math:: - H_{out} = \left\lfloor H_{in} \times \text{scale_factor} \right\rfloor + .. math:: + H_{out} = \left\lfloor H_{in} \times \text{scale\_factor} \right\rfloor - W_{out} = \left\lfloor W_{in} \times \text{scale_factor} \right\rfloor + .. math:: + W_{out} = \left\lfloor W_{in} \times \text{scale\_factor} \right\rfloor Examples:: @@ -199,10 +202,11 @@ class UpsamplingBilinear2d(Upsample): - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - .. math:: - H_{out} = \left\lfloor H_{in} \times \text{scale_factor} \right\rfloor + .. math:: + H_{out} = \left\lfloor H_{in} \times \text{scale\_factor} \right\rfloor - W_{out} = \left\lfloor W_{in} \times \text{scale_factor} \right\rfloor + .. math:: + W_{out} = \left\lfloor W_{in} \times \text{scale\_factor} \right\rfloor Examples:: diff --git a/torch/nn/utils/spectral_norm.py b/torch/nn/utils/spectral_norm.py index 1cdafb39e3830..bac43badbabf3 100644 --- a/torch/nn/utils/spectral_norm.py +++ b/torch/nn/utils/spectral_norm.py @@ -81,8 +81,8 @@ def spectral_norm(module, name='weight', n_power_iterations=1, eps=1e-12, dim=No r"""Applies spectral normalization to a parameter in the given module. .. math:: - \mathbf{W} &= \dfrac{\mathbf{W}}{\sigma(\mathbf{W})} \\ - \sigma(\mathbf{W}) &= \max_{\mathbf{h}: \mathbf{h} \ne 0} \dfrac{\|\mathbf{W} \mathbf{h}\|_2}{\|\mathbf{h}\|_2} + \mathbf{W} = \dfrac{\mathbf{W}}{\sigma(\mathbf{W})} \\ + \sigma(\mathbf{W}) = \max_{\mathbf{h}: \mathbf{h} \ne 0} \dfrac{\|\mathbf{W} \mathbf{h}\|_2}{\|\mathbf{h}\|_2} Spectral normalization stabilizes the training of discriminators (critics) in Generaive Adversarial Networks (GANs) by rescaling the weight tensor From 7be071a829f5bbaa643ccacdd20b1508f78a2d78 Mon Sep 17 00:00:00 2001 From: onnxbot Date: Thu, 2 Aug 2018 12:24:09 -0700 Subject: [PATCH 11/16] Update onnx to onnx/onnx@2a3a226 (#10167) Summary: https://github.com/onnx/onnx/commit/2a3a226a96b3dc9e50e739490dd38eaae4f2d8f3 Pull Request resolved: https://github.com/pytorch/pytorch/pull/10167 Reviewed By: houseroad Differential Revision: D9134738 Pulled By: bddppq fbshipit-source-id: 9d3fd3c04a584d5626146f174ac78cabfa0e5934 --- third_party/onnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/onnx b/third_party/onnx index 32ac71b1b9c1b..2a3a226a96b3d 160000 --- a/third_party/onnx +++ b/third_party/onnx @@ -1 +1 @@ -Subproject commit 32ac71b1b9c1bd7f196eed3b311734ec6ab3c367 +Subproject commit 2a3a226a96b3dc9e50e739490dd38eaae4f2d8f3 From 9e85a7a9de6e974922347e8a44c0b55bf044cffb Mon Sep 17 00:00:00 2001 From: Edward Yang Date: Thu, 2 Aug 2018 12:26:20 -0700 Subject: [PATCH 12/16] =?UTF-8?q?Back=20out=20"[pytorch][PR]=20[TENSOR=20M?= =?UTF-8?q?ERGE]=20Delete=20type=5F=20field=20from=20TensorImpl,=20replace?= =?UTF-8?q?d=20with=20backend=5F/scalar=5Ftyp=E2=80=A6"=20(#10169)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/10169 Original commit changeset: 2b4d867abfdc Reviewed By: pjh5, SsnL Differential Revision: D9135216 fbshipit-source-id: d5c9f12c3a0f75df224c781e1cd1e323cdfbb0d5 --- aten/src/ATen/Context.cpp | 5 +---- aten/src/ATen/SparseTensorImpl.cpp | 10 +++++----- aten/src/ATen/SparseTensorImpl.h | 2 +- aten/src/ATen/TensorImpl.cpp | 13 ------------- aten/src/ATen/TensorImpl.h | 18 ++++++------------ aten/src/ATen/UndefinedTensor.cpp | 2 +- aten/src/ATen/detail/VariableHooksInterface.h | 5 ----- aten/src/ATen/native/sparse/SparseTensor.cpp | 2 +- aten/src/ATen/templates/TensorDerived.cpp | 2 +- aten/src/TH/THTensor.hpp | 4 ---- torch/csrc/autograd/aten_variable_hooks.cpp | 5 ----- torch/csrc/autograd/variable.cpp | 6 ++---- torch/csrc/autograd/variable.h | 3 +++ torch/csrc/jit/interpreter.cpp | 2 +- 14 files changed, 22 insertions(+), 57 deletions(-) diff --git a/aten/src/ATen/Context.cpp b/aten/src/ATen/Context.cpp index d153e6bc6ada0..59f6ff755ee3f 100644 --- a/aten/src/ATen/Context.cpp +++ b/aten/src/ATen/Context.cpp @@ -37,11 +37,8 @@ Context::Context() Type::registerCPU(this); } -// NB: Ensure that globalContext is initialized before we load -// variable hooks, otherwise we will deadlock. Regardless, the -// deadlock is bad, and being tracked at https://github.com/pytorch/pytorch/issues/9784 -static Context globalContext_; Context & globalContext() { + static Context globalContext_; return globalContext_; } diff --git a/aten/src/ATen/SparseTensorImpl.cpp b/aten/src/ATen/SparseTensorImpl.cpp index 03a5a6008e7d2..968fd8ebbec26 100644 --- a/aten/src/ATen/SparseTensorImpl.cpp +++ b/aten/src/ATen/SparseTensorImpl.cpp @@ -18,14 +18,14 @@ namespace at { // tensor and a [0] size values tensor for such an empty tensor. However, // we don't currently support zero-size dimensions, so we can't actually // do this; so we just allocate zero-size tensors for everything. -SparseTensorImpl::SparseTensorImpl(at::Backend backend, at::ScalarType scalar_type) - : TensorImpl(backend, scalar_type, nullptr, false) +SparseTensorImpl::SparseTensorImpl(Type * type) + : TensorImpl(type, nullptr) , size_{0} , sparseDims_(1) , denseDims_(0) - , indices_(globalContext().getTypeOpt(toDense(backend), ScalarType::Long)->tensor()) - , values_(globalContext().getTypeOpt(toDense(backend), scalar_type)->tensor()) { - AT_ASSERT(backend == Backend::SparseCPU || backend == Backend::SparseCUDA); + , indices_(type->toDense().toScalarType(ScalarType::Long).tensor()) + , values_(type->toDense().tensor()) { + AT_ASSERT(type->is_sparse()); } IntList SparseTensorImpl::sizes() const { diff --git a/aten/src/ATen/SparseTensorImpl.h b/aten/src/ATen/SparseTensorImpl.h index 307c0f9e5574d..de0f843132986 100644 --- a/aten/src/ATen/SparseTensorImpl.h +++ b/aten/src/ATen/SparseTensorImpl.h @@ -48,7 +48,7 @@ struct AT_API SparseTensorImpl : public TensorImpl { public: // Public for now... - explicit SparseTensorImpl(at::Backend, at::ScalarType); + explicit SparseTensorImpl(Type * type); int64_t nnz() const { return nnz_; } int64_t sparseDims() const { return sparseDims_; } diff --git a/aten/src/ATen/TensorImpl.cpp b/aten/src/ATen/TensorImpl.cpp index a48cb033b2de4..59cc303a1acf5 100644 --- a/aten/src/ATen/TensorImpl.cpp +++ b/aten/src/ATen/TensorImpl.cpp @@ -2,23 +2,10 @@ #include #include -#include - -#include #include namespace at { - -Type& TensorImpl::type() const { - Type* base_type = &globalContext().getType(backend_, scalar_type_); - if (is_variable_) { - return detail::getVariableHooks().getVariableType(*base_type); - } else { - return *base_type; - } -} - Tensor& TensorImpl::grad() { AT_ERROR("grad is not implemented for Tensor"); } diff --git a/aten/src/ATen/TensorImpl.h b/aten/src/ATen/TensorImpl.h index 1aa4d8390ed17..9c3591eb96b31 100644 --- a/aten/src/ATen/TensorImpl.h +++ b/aten/src/ATen/TensorImpl.h @@ -18,18 +18,16 @@ struct Tensor; namespace at { struct AT_API TensorImpl : public Retainable { - explicit TensorImpl(Backend backend, ScalarType scalar_type, THTensor * tensor, bool is_variable) - : backend_(backend), scalar_type_(scalar_type), is_variable_(is_variable), tensor(tensor) {} + explicit TensorImpl(Type * type, THTensor * tensor) + : type_(type), tensor(tensor) {} virtual ~TensorImpl(); virtual void release_resources() override; - // The implementation of this method will have to be hoisted out and - // hooked in, so that Caffe2 doesn't need to know about Context - // TODO: This really really needs to be inlined. - Type & type() const; - + Type & type() const { + return *type_; + } const char * toString() const; virtual IntList sizes() const; virtual IntList strides() const; @@ -93,12 +91,8 @@ struct AT_API TensorImpl : public Retainable { virtual void set_data(Tensor new_data); protected: - Backend backend_; - // INVARIANT: When storage is non-null, this scalar type must - // agree with the scalar type in storage - ScalarType scalar_type_; - bool is_variable_ = false; bool is_wrapped_number_ = false; + Type * type_; public: THTensor * tensor; }; diff --git a/aten/src/ATen/UndefinedTensor.cpp b/aten/src/ATen/UndefinedTensor.cpp index ecfb70fa1bbed..5e4059421c128 100644 --- a/aten/src/ATen/UndefinedTensor.cpp +++ b/aten/src/ATen/UndefinedTensor.cpp @@ -6,7 +6,7 @@ namespace at { // should this use the globalContext? Can it get a context passed in somehow? UndefinedTensor::UndefinedTensor() -: TensorImpl(Backend::Undefined, ScalarType::Undefined, nullptr, /* is variable */ false) { +: TensorImpl(&(globalContext().getType(Backend::Undefined,ScalarType::Undefined)), nullptr) { } IntList UndefinedTensor::sizes() const { diff --git a/aten/src/ATen/detail/VariableHooksInterface.h b/aten/src/ATen/detail/VariableHooksInterface.h index 836dacb97766e..287116490397f 100644 --- a/aten/src/ATen/detail/VariableHooksInterface.h +++ b/aten/src/ATen/detail/VariableHooksInterface.h @@ -3,7 +3,6 @@ #include #include #include -#include namespace at { class Context; @@ -26,10 +25,6 @@ struct AT_API VariableHooksInterface { // squelch -Werror=non-virtual-dtor virtual ~VariableHooksInterface() {} - virtual Type& getVariableType(const at::Type& baseType) const { - AT_ERROR("cannot getVariableType without libtorch"); - } - virtual void registerVariableTypeFor(Context*, Backend backend, ScalarType scalar_type) const { // no-op if Variable not available; it'll get handled (if at all) when // libtorch.so gets loaded diff --git a/aten/src/ATen/native/sparse/SparseTensor.cpp b/aten/src/ATen/native/sparse/SparseTensor.cpp index 7a7e8be5c7ff6..0cac9bcb9131f 100644 --- a/aten/src/ATen/native/sparse/SparseTensor.cpp +++ b/aten/src/ATen/native/sparse/SparseTensor.cpp @@ -63,7 +63,7 @@ SparseTensor new_sparse(const SparseType& dtype) { AT_ASSERT(!dtype.is_variable()); AT_ASSERT(dtype.is_sparse()); // TODO: Hmm... this const_cast business seems a bit dodgy - return SparseTensor(new SparseTensorImpl(dtype.backend(), dtype.scalarType()), /* retain */ false); + return SparseTensor(new SparseTensorImpl(const_cast(&dtype)), /* retain */ false); } /*** Helper methods ***/ diff --git a/aten/src/ATen/templates/TensorDerived.cpp b/aten/src/ATen/templates/TensorDerived.cpp index 5fab8bf222641..7ad922321a22c 100644 --- a/aten/src/ATen/templates/TensorDerived.cpp +++ b/aten/src/ATen/templates/TensorDerived.cpp @@ -21,7 +21,7 @@ namespace detail { } ${Tensor}::${Tensor}(${THTensor} * tensor) -: TensorImpl(Backend::${Backend}, ScalarType::${ScalarName}, tensor, /* is variable */ false) +: TensorImpl(&globalContext().getType(Backend::${Backend},ScalarType::${ScalarName}), tensor) {} ${TensorDenseOrSparse} diff --git a/aten/src/TH/THTensor.hpp b/aten/src/TH/THTensor.hpp index 9b1584f9c342e..8504b454f12fb 100644 --- a/aten/src/TH/THTensor.hpp +++ b/aten/src/TH/THTensor.hpp @@ -56,10 +56,6 @@ struct THTensor return sizes_.size(); } - at::ScalarType scalar_type() const { - return storage_->scalar_type; - } - ptrdiff_t storage_offset() const { return storage_offset_; } diff --git a/torch/csrc/autograd/aten_variable_hooks.cpp b/torch/csrc/autograd/aten_variable_hooks.cpp index 2f3899e4f8b59..7a2c3974c2227 100644 --- a/torch/csrc/autograd/aten_variable_hooks.cpp +++ b/torch/csrc/autograd/aten_variable_hooks.cpp @@ -6,7 +6,6 @@ namespace torch { namespace autograd { struct VariableHooks : public at::VariableHooksInterface { VariableHooks(at::VariableHooksArgs) {} void registerVariableTypeFor(at::Context*, at::Backend, at::ScalarType) const override; - at::Type& getVariableType(const at::Type&) const override; }; // Sigh, the registry doesn't support namespaces :( @@ -21,8 +20,4 @@ void VariableHooks::registerVariableTypeFor(at::Context* context, at::Backend ba register_variable_type_for(baseType); } -at::Type& VariableHooks::getVariableType(const at::Type& baseType) const { - return *VariableType::getType(baseType); -} - }} // torch::autograd diff --git a/torch/csrc/autograd/variable.cpp b/torch/csrc/autograd/variable.cpp index 30aded0a85e73..966abb523a7c8 100644 --- a/torch/csrc/autograd/variable.cpp +++ b/torch/csrc/autograd/variable.cpp @@ -22,7 +22,7 @@ namespace torch { namespace autograd { Variable::Impl::Impl(at::Tensor data, bool requires_grad, Edge gradient_edge) - : TensorImpl(data.type().backend(), data.type().scalarType(), nullptr, /* is variable */ true), + : TensorImpl(VariableType::getType(data), nullptr), data_(std::move(data)), grad_fn_(std::move(gradient_edge.function)), requires_grad_(false), @@ -118,9 +118,7 @@ void Variable::Impl::backward( void Variable::Impl::set_data(Tensor new_data) { if (new_data.type() != data_.type()) { - scalar_type_ = new_data.type().scalarType(); - backend_ = new_data.type().backend(); - is_variable_ = true; + type_ = VariableType::getType(new_data.type()); // Clear grad_accumulator if it exists, since it stores the old type info. grad_accumulator_.reset(); } diff --git a/torch/csrc/autograd/variable.h b/torch/csrc/autograd/variable.h index d46008bbdd10b..633b8028f765d 100644 --- a/torch/csrc/autograd/variable.h +++ b/torch/csrc/autograd/variable.h @@ -327,6 +327,9 @@ struct Variable::Impl : public at::TensorImpl { /// Reset all expensive fields to free up resources void release_resources() override; + // Make this field public so we can access it from `Variable`. + using at::TensorImpl::type_; + std::string name; at::Tensor data_; diff --git a/torch/csrc/jit/interpreter.cpp b/torch/csrc/jit/interpreter.cpp index 0c1fe17ade0df..da6f629d629e4 100644 --- a/torch/csrc/jit/interpreter.cpp +++ b/torch/csrc/jit/interpreter.cpp @@ -337,7 +337,7 @@ struct PreprocessGraph { struct ContainerTensor : public at::TensorImpl { public: ContainerTensor() - : TensorImpl(at::Backend::Undefined,at::ScalarType::Undefined, nullptr, /* is_variable */ false) {} + : TensorImpl(&(at::globalContext().getType(at::Backend::Undefined,at::ScalarType::Undefined)), nullptr) {} virtual ~ContainerTensor() = default; virtual at::IntList sizes() const override { From 538b15d13c368b748f0b3d1bd4984a34b20d591a Mon Sep 17 00:00:00 2001 From: Tongzhou Wang Date: Thu, 2 Aug 2018 12:49:30 -0700 Subject: [PATCH 13/16] Use PYTORCH_PYTHON to call generate_code.py (#10171) Summary: Probably fixes https://github.com/pytorch/pytorch/issues/8373#issuecomment-409994847 Pull Request resolved: https://github.com/pytorch/pytorch/pull/10171 Differential Revision: D9135607 Pulled By: SsnL fbshipit-source-id: 72f535875658c857621e41fd25c2174052714557 --- torch/CMakeLists.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/torch/CMakeLists.txt b/torch/CMakeLists.txt index d64073dba721c..b41b7898e267b 100644 --- a/torch/CMakeLists.txt +++ b/torch/CMakeLists.txt @@ -46,6 +46,13 @@ if("${isSystemDir}" STREQUAL "-1") list(APPEND CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") endif() +# Get the correct Python executable +if (DEFINED ENV{PYTORCH_PYTHON}) + message(STATUS "Using python found in $ENV{PYTORCH_PYTHON}") + set(PYCMD "$ENV{PYTORCH_PYTHON}") +else() + SET(PYCMD "python") +endif() # Generate files set(TOOLS_PATH "${TORCH_SRC_DIR}/../tools") @@ -79,7 +86,7 @@ add_custom_command( "${TORCH_SRC_DIR}/csrc/jit/generated/register_aten_ops.cpp" "${TORCH_SRC_DIR}/csrc/jit/generated/aten_interned_strings.h" COMMAND - python tools/setup_helpers/generate_code.py + ${PYCMD} tools/setup_helpers/generate_code.py --declarations-path "${CMAKE_BINARY_DIR}/aten/src/ATen/Declarations.yaml" --nn-path "aten/src/" DEPENDS From 94c67f1454e7ed5cbcc2c96d16c8618ff2e66367 Mon Sep 17 00:00:00 2001 From: Christian Puhrsch Date: Thu, 2 Aug 2018 13:20:12 -0700 Subject: [PATCH 14/16] Replace storageimpl type with scalar_type and backend Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/10097 Differential Revision: D9124287 Pulled By: cpuhrsch fbshipit-source-id: c976abeeaaa085b972812c1a3270eb6aef0c0dca --- aten/src/ATen/StorageImpl.cpp | 11 ++++++----- aten/src/ATen/StorageImpl.h | 5 +++-- aten/src/ATen/Utils.h | 9 +++++---- torch/csrc/DynamicTypes.cpp | 4 +++- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/aten/src/ATen/StorageImpl.cpp b/aten/src/ATen/StorageImpl.cpp index 6e3d693d012c5..35f1d8076afae 100644 --- a/aten/src/ATen/StorageImpl.cpp +++ b/aten/src/ATen/StorageImpl.cpp @@ -28,11 +28,12 @@ StorageImpl::StorageImpl( allocator, resizable) {} -Type& StorageImpl::type() { - if (data_ptr.device().is_cuda()) { - return globalContext().getType(Backend::CUDA, scalar_type); +namespace detail { +Backend get_backend(StorageImpl* storage_impl) { + if (storage_impl->data_ptr.device().is_cuda()) { + return Backend::CUDA; } - return globalContext().getType(Backend::CPU, scalar_type); + return Backend::CPU; } - +} // namespace detail } // namespace at diff --git a/aten/src/ATen/StorageImpl.h b/aten/src/ATen/StorageImpl.h index f1c23c54677db..c6c737e1b851b 100644 --- a/aten/src/ATen/StorageImpl.h +++ b/aten/src/ATen/StorageImpl.h @@ -91,8 +91,6 @@ struct AT_API StorageImpl : public Retainable { return at::elementSize(scalar_type); } - Type& type(); - //TODO: Rename to size() and size to size_ size_t get_size() const { return size; @@ -112,4 +110,7 @@ struct AT_API StorageImpl : public Retainable { } }; +namespace detail { +AT_API Backend get_backend(StorageImpl* storage_impl); +} } // namespace at diff --git a/aten/src/ATen/Utils.h b/aten/src/ATen/Utils.h index 437b001a1c168..be2f180c075d2 100644 --- a/aten/src/ATen/Utils.h +++ b/aten/src/ATen/Utils.h @@ -1,6 +1,7 @@ #pragma once #include "ATen/ATenGeneral.h" +#include "ATen/StorageImpl.h" #include "ATen/ArrayRef.h" #include "ATen/Error.h" #include "ATen/UndefinedTensor.h" @@ -24,12 +25,12 @@ AT_API int _crash_if_asan(int); template static inline T* checked_cast_storage(Base* expr, const char * name, int pos, Backend backend, ScalarType scalar_type) { - if (expr->pImpl()->type().backend() != backend) { - AT_ERROR("Expected object of backend ", backend, " but got backend ", expr->pImpl()->type().backend(), + if (at::detail::get_backend(expr->pImpl()) != backend) { + AT_ERROR("Expected object of backend ", backend, " but got backend ", at::detail::get_backend(expr->pImpl()), " for argument #", pos, " '", name, "'"); } - if (expr->pImpl()->type().scalarType() != scalar_type) { - AT_ERROR("Expected object of scalar type ", scalar_type, " but got scalar type ", expr->pImpl()->type().scalarType(), + if (expr->pImpl()->scalar_type != scalar_type) { + AT_ERROR("Expected object of scalar type ", scalar_type, " but got scalar type ", expr->pImpl()->scalar_type, " for argument #", pos, " '", name, "'"); } // NB: We're getting rid of derived types soon! diff --git a/torch/csrc/DynamicTypes.cpp b/torch/csrc/DynamicTypes.cpp index 56be1f731572d..a83a4aa291f35 100644 --- a/torch/csrc/DynamicTypes.cpp +++ b/torch/csrc/DynamicTypes.cpp @@ -69,7 +69,9 @@ at::Type* get_type(const std::string& name, bool is_cuda, bool is_sparse) { PyTypeObject* getPyTypeObject(const at::Storage& storage) { - auto it = attype_to_py_storage_type.find(&storage.pImpl()->type()); + auto attype = at::globalContext().getTypeOpt( + at::detail::get_backend(storage.pImpl()), storage.pImpl()->scalar_type); + auto it = attype_to_py_storage_type.find(attype); if (it != attype_to_py_storage_type.end()) { return it->second; } From dad6e8bb6c6be1843f685842905761a47106b19b Mon Sep 17 00:00:00 2001 From: Owen Anderson Date: Thu, 2 Aug 2018 13:35:13 -0700 Subject: [PATCH 15/16] Remove capture specifiers in register_aten_ops when they're not needed. (#9669) Summary: zdevito Pull Request resolved: https://github.com/pytorch/pytorch/pull/9669 Differential Revision: D8952335 Pulled By: resistor fbshipit-source-id: 8fbbec7a7f55fbeeda3509cb3d339e1db90a53e6 --- tools/jit/gen_jit_dispatch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/jit/gen_jit_dispatch.py b/tools/jit/gen_jit_dispatch.py index 5a76d447ad249..d337143dd8b09 100644 --- a/tools/jit/gen_jit_dispatch.py +++ b/tools/jit/gen_jit_dispatch.py @@ -191,6 +191,7 @@ def emit_decl_variant(decl): kw_assignments = [] arguments = [] num_inputs = len(decl['arguments']) + op_capture = '' real_inputs = 0 for arg in decl['arguments']: @@ -208,7 +209,8 @@ def emit_decl_variant(decl): constructor = CONSTRUCTOR.substitute(name=decl['name'], call=call, kw_assignments=kw_assignments, - num_inputs=num_inputs) + num_inputs=num_inputs, + op_capture=op_capture) return constructor # This function declares an order on declarations. This is necessary because From 7dc870bd7b138222409d947e8c87cd6b2d5ef346 Mon Sep 17 00:00:00 2001 From: Taewook Oh Date: Thu, 2 Aug 2018 14:43:59 -0700 Subject: [PATCH 16/16] Delete invalid 'template' keyword (#10173) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/10173 With D9024330, `Extend` fundtion is no more a template, which makes the `template` keyword here invalid. For some reason current version of LLVM doesn't catch this, but the latest one does. Reviewed By: jerryzh168 Differential Revision: D9133462 fbshipit-source-id: 54ac9aad01f81b9b4e7b6e2864b8961478d2d860 --- caffe2/experiments/operators/tt_pad_op.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caffe2/experiments/operators/tt_pad_op.h b/caffe2/experiments/operators/tt_pad_op.h index c85101542ce4f..6d4ab57a9d66a 100644 --- a/caffe2/experiments/operators/tt_pad_op.h +++ b/caffe2/experiments/operators/tt_pad_op.h @@ -52,7 +52,7 @@ class TTPadOp final : public Operator { TIndex padded_dim0 = (X_dim0 / scale_ + 1) * scale_; auto dim0_diff = padded_dim0 - X_dim0; // set growthPct to the upper bound percentage: (100 * scale_ / X_dim0) - X_pad->template Extend(dim0_diff, 100 * scale_ / X_dim0, &context_); + X_pad->Extend(dim0_diff, 100 * scale_ / X_dim0, &context_); auto* X_pad_data = X_pad->template mutable_data(); TIndex X_size = X_dim0 * X_dim1;