diff --git a/.jenkins/pytorch/build.sh b/.jenkins/pytorch/build.sh index 48e81dfd635bc..cc8d0991f7218 100755 --- a/.jenkins/pytorch/build.sh +++ b/.jenkins/pytorch/build.sh @@ -32,6 +32,7 @@ pip install -r requirements.txt || true if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then # This is necessary in order to cross compile (or else we'll have missing GPU device). export MAX_JOBS=4 + # This is necessary in order to cross compile (or else we'll have missing GPU device). export HCC_AMDGPU_TARGET=gfx900 # These environment variables are not set on CI when we were running as the Jenkins user. @@ -42,7 +43,6 @@ if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then # This environment variable enabled HCC Optimizations that speed up the linking stage. # https://github.com/RadeonOpenCompute/hcc#hcc-with-thinlto-linking export KMTHINLTO=1 - python tools/amd_build/build_pytorch_amd.py USE_ROCM=1 python setup.py install --user exit 0 diff --git a/aten/src/ATen/Context.cpp b/aten/src/ATen/Context.cpp index d153e6bc6ada0..59f6ff755ee3f 100644 --- a/aten/src/ATen/Context.cpp +++ b/aten/src/ATen/Context.cpp @@ -37,11 +37,8 @@ Context::Context() Type::registerCPU(this); } -// NB: Ensure that globalContext is initialized before we load -// variable hooks, otherwise we will deadlock. Regardless, the -// deadlock is bad, and being tracked at https://github.com/pytorch/pytorch/issues/9784 -static Context globalContext_; Context & globalContext() { + static Context globalContext_; return globalContext_; } diff --git a/aten/src/ATen/SparseTensorImpl.cpp b/aten/src/ATen/SparseTensorImpl.cpp index 03a5a6008e7d2..968fd8ebbec26 100644 --- a/aten/src/ATen/SparseTensorImpl.cpp +++ b/aten/src/ATen/SparseTensorImpl.cpp @@ -18,14 +18,14 @@ namespace at { // tensor and a [0] size values tensor for such an empty tensor. However, // we don't currently support zero-size dimensions, so we can't actually // do this; so we just allocate zero-size tensors for everything. -SparseTensorImpl::SparseTensorImpl(at::Backend backend, at::ScalarType scalar_type) - : TensorImpl(backend, scalar_type, nullptr, false) +SparseTensorImpl::SparseTensorImpl(Type * type) + : TensorImpl(type, nullptr) , size_{0} , sparseDims_(1) , denseDims_(0) - , indices_(globalContext().getTypeOpt(toDense(backend), ScalarType::Long)->tensor()) - , values_(globalContext().getTypeOpt(toDense(backend), scalar_type)->tensor()) { - AT_ASSERT(backend == Backend::SparseCPU || backend == Backend::SparseCUDA); + , indices_(type->toDense().toScalarType(ScalarType::Long).tensor()) + , values_(type->toDense().tensor()) { + AT_ASSERT(type->is_sparse()); } IntList SparseTensorImpl::sizes() const { diff --git a/aten/src/ATen/SparseTensorImpl.h b/aten/src/ATen/SparseTensorImpl.h index 307c0f9e5574d..de0f843132986 100644 --- a/aten/src/ATen/SparseTensorImpl.h +++ b/aten/src/ATen/SparseTensorImpl.h @@ -48,7 +48,7 @@ struct AT_API SparseTensorImpl : public TensorImpl { public: // Public for now... - explicit SparseTensorImpl(at::Backend, at::ScalarType); + explicit SparseTensorImpl(Type * type); int64_t nnz() const { return nnz_; } int64_t sparseDims() const { return sparseDims_; } diff --git a/aten/src/ATen/StorageImpl.cpp b/aten/src/ATen/StorageImpl.cpp index 6e3d693d012c5..35f1d8076afae 100644 --- a/aten/src/ATen/StorageImpl.cpp +++ b/aten/src/ATen/StorageImpl.cpp @@ -28,11 +28,12 @@ StorageImpl::StorageImpl( allocator, resizable) {} -Type& StorageImpl::type() { - if (data_ptr.device().is_cuda()) { - return globalContext().getType(Backend::CUDA, scalar_type); +namespace detail { +Backend get_backend(StorageImpl* storage_impl) { + if (storage_impl->data_ptr.device().is_cuda()) { + return Backend::CUDA; } - return globalContext().getType(Backend::CPU, scalar_type); + return Backend::CPU; } - +} // namespace detail } // namespace at diff --git a/aten/src/ATen/StorageImpl.h b/aten/src/ATen/StorageImpl.h index f1c23c54677db..c6c737e1b851b 100644 --- a/aten/src/ATen/StorageImpl.h +++ b/aten/src/ATen/StorageImpl.h @@ -91,8 +91,6 @@ struct AT_API StorageImpl : public Retainable { return at::elementSize(scalar_type); } - Type& type(); - //TODO: Rename to size() and size to size_ size_t get_size() const { return size; @@ -112,4 +110,7 @@ struct AT_API StorageImpl : public Retainable { } }; +namespace detail { +AT_API Backend get_backend(StorageImpl* storage_impl); +} } // namespace at diff --git a/aten/src/ATen/TensorImpl.cpp b/aten/src/ATen/TensorImpl.cpp index a48cb033b2de4..59cc303a1acf5 100644 --- a/aten/src/ATen/TensorImpl.cpp +++ b/aten/src/ATen/TensorImpl.cpp @@ -2,23 +2,10 @@ #include #include -#include - -#include #include namespace at { - -Type& TensorImpl::type() const { - Type* base_type = &globalContext().getType(backend_, scalar_type_); - if (is_variable_) { - return detail::getVariableHooks().getVariableType(*base_type); - } else { - return *base_type; - } -} - Tensor& TensorImpl::grad() { AT_ERROR("grad is not implemented for Tensor"); } diff --git a/aten/src/ATen/TensorImpl.h b/aten/src/ATen/TensorImpl.h index 1aa4d8390ed17..9c3591eb96b31 100644 --- a/aten/src/ATen/TensorImpl.h +++ b/aten/src/ATen/TensorImpl.h @@ -18,18 +18,16 @@ struct Tensor; namespace at { struct AT_API TensorImpl : public Retainable { - explicit TensorImpl(Backend backend, ScalarType scalar_type, THTensor * tensor, bool is_variable) - : backend_(backend), scalar_type_(scalar_type), is_variable_(is_variable), tensor(tensor) {} + explicit TensorImpl(Type * type, THTensor * tensor) + : type_(type), tensor(tensor) {} virtual ~TensorImpl(); virtual void release_resources() override; - // The implementation of this method will have to be hoisted out and - // hooked in, so that Caffe2 doesn't need to know about Context - // TODO: This really really needs to be inlined. - Type & type() const; - + Type & type() const { + return *type_; + } const char * toString() const; virtual IntList sizes() const; virtual IntList strides() const; @@ -93,12 +91,8 @@ struct AT_API TensorImpl : public Retainable { virtual void set_data(Tensor new_data); protected: - Backend backend_; - // INVARIANT: When storage is non-null, this scalar type must - // agree with the scalar type in storage - ScalarType scalar_type_; - bool is_variable_ = false; bool is_wrapped_number_ = false; + Type * type_; public: THTensor * tensor; }; diff --git a/aten/src/ATen/UndefinedTensor.cpp b/aten/src/ATen/UndefinedTensor.cpp index ecfb70fa1bbed..5e4059421c128 100644 --- a/aten/src/ATen/UndefinedTensor.cpp +++ b/aten/src/ATen/UndefinedTensor.cpp @@ -6,7 +6,7 @@ namespace at { // should this use the globalContext? Can it get a context passed in somehow? UndefinedTensor::UndefinedTensor() -: TensorImpl(Backend::Undefined, ScalarType::Undefined, nullptr, /* is variable */ false) { +: TensorImpl(&(globalContext().getType(Backend::Undefined,ScalarType::Undefined)), nullptr) { } IntList UndefinedTensor::sizes() const { diff --git a/aten/src/ATen/Utils.h b/aten/src/ATen/Utils.h index 437b001a1c168..be2f180c075d2 100644 --- a/aten/src/ATen/Utils.h +++ b/aten/src/ATen/Utils.h @@ -1,6 +1,7 @@ #pragma once #include "ATen/ATenGeneral.h" +#include "ATen/StorageImpl.h" #include "ATen/ArrayRef.h" #include "ATen/Error.h" #include "ATen/UndefinedTensor.h" @@ -24,12 +25,12 @@ AT_API int _crash_if_asan(int); template static inline T* checked_cast_storage(Base* expr, const char * name, int pos, Backend backend, ScalarType scalar_type) { - if (expr->pImpl()->type().backend() != backend) { - AT_ERROR("Expected object of backend ", backend, " but got backend ", expr->pImpl()->type().backend(), + if (at::detail::get_backend(expr->pImpl()) != backend) { + AT_ERROR("Expected object of backend ", backend, " but got backend ", at::detail::get_backend(expr->pImpl()), " for argument #", pos, " '", name, "'"); } - if (expr->pImpl()->type().scalarType() != scalar_type) { - AT_ERROR("Expected object of scalar type ", scalar_type, " but got scalar type ", expr->pImpl()->type().scalarType(), + if (expr->pImpl()->scalar_type != scalar_type) { + AT_ERROR("Expected object of scalar type ", scalar_type, " but got scalar type ", expr->pImpl()->scalar_type, " for argument #", pos, " '", name, "'"); } // NB: We're getting rid of derived types soon! diff --git a/aten/src/ATen/detail/VariableHooksInterface.h b/aten/src/ATen/detail/VariableHooksInterface.h index 836dacb97766e..287116490397f 100644 --- a/aten/src/ATen/detail/VariableHooksInterface.h +++ b/aten/src/ATen/detail/VariableHooksInterface.h @@ -3,7 +3,6 @@ #include #include #include -#include namespace at { class Context; @@ -26,10 +25,6 @@ struct AT_API VariableHooksInterface { // squelch -Werror=non-virtual-dtor virtual ~VariableHooksInterface() {} - virtual Type& getVariableType(const at::Type& baseType) const { - AT_ERROR("cannot getVariableType without libtorch"); - } - virtual void registerVariableTypeFor(Context*, Backend backend, ScalarType scalar_type) const { // no-op if Variable not available; it'll get handled (if at all) when // libtorch.so gets loaded diff --git a/aten/src/ATen/native/sparse/SparseTensor.cpp b/aten/src/ATen/native/sparse/SparseTensor.cpp index 7a7e8be5c7ff6..0cac9bcb9131f 100644 --- a/aten/src/ATen/native/sparse/SparseTensor.cpp +++ b/aten/src/ATen/native/sparse/SparseTensor.cpp @@ -63,7 +63,7 @@ SparseTensor new_sparse(const SparseType& dtype) { AT_ASSERT(!dtype.is_variable()); AT_ASSERT(dtype.is_sparse()); // TODO: Hmm... this const_cast business seems a bit dodgy - return SparseTensor(new SparseTensorImpl(dtype.backend(), dtype.scalarType()), /* retain */ false); + return SparseTensor(new SparseTensorImpl(const_cast(&dtype)), /* retain */ false); } /*** Helper methods ***/ diff --git a/aten/src/ATen/templates/TensorDerived.cpp b/aten/src/ATen/templates/TensorDerived.cpp index 5fab8bf222641..7ad922321a22c 100644 --- a/aten/src/ATen/templates/TensorDerived.cpp +++ b/aten/src/ATen/templates/TensorDerived.cpp @@ -21,7 +21,7 @@ namespace detail { } ${Tensor}::${Tensor}(${THTensor} * tensor) -: TensorImpl(Backend::${Backend}, ScalarType::${ScalarName}, tensor, /* is variable */ false) +: TensorImpl(&globalContext().getType(Backend::${Backend},ScalarType::${ScalarName}), tensor) {} ${TensorDenseOrSparse} diff --git a/aten/src/TH/THTensor.hpp b/aten/src/TH/THTensor.hpp index 56204a00e9c3e..8504b454f12fb 100644 --- a/aten/src/TH/THTensor.hpp +++ b/aten/src/TH/THTensor.hpp @@ -56,10 +56,6 @@ struct THTensor return sizes_.size(); } - at::ScalarType scalar_type() const { - return storage_->scalar_type; - } - ptrdiff_t storage_offset() const { return storage_offset_; } @@ -127,41 +123,6 @@ inline THStorage* THTensor_getStoragePtr(const THTensor* tensor) { return tensor->storage_; } -#include "generic/THTensorFastGetSet.hpp" -#include "THGenerateAllTypes.h" - -inline void THTensor_resizeDim(THTensor* tensor, int64_t ndim) { - // NB: This is *truly* a resize; calling code (e.g., squeeze) - // assumes that old values are preserved - tensor->is_zero_dim_ = bool(ndim == 0); - tensor->sizes_.resize(ndim); - tensor->strides_.resize(ndim); -} - -inline void THTensor_setSizesAndStrides(THTensor* tensor, std::vector&& new_size, std::vector&& new_stride) { - tensor->sizes_ = std::move(new_size); - tensor->strides_ = std::move(new_stride); -} - -inline void THTensor_setSizeAtDim(THTensor* tensor, int dim, int64_t new_size) { - tensor->sizes_[dim] = new_size; -} - -inline void THTensor_setStrideAtDim(THTensor* tensor, int dim, int64_t new_stride) { - tensor->strides_[dim] = new_stride; -} - -inline void THTensor_setStorageOffset(THTensor* tensor, ptrdiff_t storage_offset) { - tensor->storage_offset_ = storage_offset; -} - -// NB: Steals ownership of storage -inline void THTensor_stealAndSetStoragePtr(THTensor* tensor, THStorage* storage) { - // Caffe2 might have tensors whose storages are null, but we - // don't allow it in PyTorch. - AT_ASSERT(storage); - tensor->storage_ = storage; -} inline bool THTensor_isZeroDim(const THTensor *tensor) { return tensor->is_zero_dim_; @@ -209,6 +170,42 @@ inline int64_t THTensor_sizeLegacyNoScalars(const THTensor *self, int dim) return THTensor_isZeroDim(self) ? 1 : self->size(dim); } +#include "generic/THTensorFastGetSet.hpp" +#include "THGenerateAllTypes.h" + +inline void THTensor_resizeDim(THTensor* tensor, int64_t ndim) { + // NB: This is *truly* a resize; calling code (e.g., squeeze) + // assumes that old values are preserved + tensor->is_zero_dim_ = bool(ndim == 0); + tensor->sizes_.resize(ndim); + tensor->strides_.resize(ndim); +} + +inline void THTensor_setSizesAndStrides(THTensor* tensor, std::vector&& new_size, std::vector&& new_stride) { + tensor->sizes_ = std::move(new_size); + tensor->strides_ = std::move(new_stride); +} + +inline void THTensor_setSizeAtDim(THTensor* tensor, int dim, int64_t new_size) { + tensor->sizes_[dim] = new_size; +} + +inline void THTensor_setStrideAtDim(THTensor* tensor, int dim, int64_t new_stride) { + tensor->strides_[dim] = new_stride; +} + +inline void THTensor_setStorageOffset(THTensor* tensor, ptrdiff_t storage_offset) { + tensor->storage_offset_ = storage_offset; +} + +// NB: Steals ownership of storage +inline void THTensor_stealAndSetStoragePtr(THTensor* tensor, THStorage* storage) { + // Caffe2 might have tensors whose storages are null, but we + // don't allow it in PyTorch. + AT_ASSERT(storage); + tensor->storage_ = storage; +} + TH_API void THTensor_free(THTensor *self); TH_CPP_API at::optional> THTensor_compute_stride(at::IntList oldshape, at::IntList oldstride, at::IntList newshape); diff --git a/aten/src/TH/THTensorApply.h b/aten/src/TH/THTensorApply.h index 9b80323a238ed..144c7c67797a3 100644 --- a/aten/src/TH/THTensorApply.h +++ b/aten/src/TH/THTensorApply.h @@ -47,9 +47,9 @@ TENSOR##_size = 1; \ TENSOR##_stride = 1; \ for(TENSOR##_i = THTensor_nDimensionLegacyAll(TENSOR)-1; TENSOR##_i >= 0; TENSOR##_i--) { \ - if(TENSOR->size(TENSOR##_i) != 1) { \ - if(TENSOR->stride(TENSOR##_i) == TENSOR##_size && TENSOR##_i != DIM) \ - TENSOR##_size *= TENSOR->size(TENSOR##_i); \ + if(THTensor_sizeLegacyNoScalars(TENSOR, TENSOR##_i) != 1) { \ + if(THTensor_strideLegacyNoScalars(TENSOR, TENSOR##_i) == TENSOR##_size && TENSOR##_i != DIM) \ + TENSOR##_size *= THTensor_sizeLegacyNoScalars(TENSOR, TENSOR##_i); \ else{ \ TENSOR##_contiguous = 0; \ break; \ @@ -70,8 +70,8 @@ TENSOR##_strides = TENSOR##_counter + 2*TENSOR##_dim; \ TH_TENSOR_dim_index = TENSOR##_dim-1; \ TENSOR##_dimOffset = (DIM == THTensor_nDimensionLegacyAll(TENSOR)-1) ? &TENSOR##_i : &TENSOR##_counter[DIM]; \ - TENSOR##_sizes[TH_TENSOR_dim_index] = TENSOR->size(THTensor_nDimensionLegacyAll(TENSOR)-1); \ - TENSOR##_strides[TH_TENSOR_dim_index] = TENSOR->stride(THTensor_nDimensionLegacyAll(TENSOR)-1); \ + TENSOR##_sizes[TH_TENSOR_dim_index] = THTensor_sizeLegacyNoScalars(TENSOR, THTensor_nDimensionLegacyAll(TENSOR)-1); \ + TENSOR##_strides[TH_TENSOR_dim_index] = THTensor_strideLegacyNoScalars(TENSOR, THTensor_nDimensionLegacyAll(TENSOR)-1); \ /* TENSOR##_counter tracks where we are in the storage. The offset into the */ \ /* storage is given by storage_offset + (i * j), where i is the stride */ \ /* vector and j is tensor_counter vector. This sets the starting position for the loop. */ \ diff --git a/aten/src/TH/THTensorDimApply.h b/aten/src/TH/THTensorDimApply.h index ff05ed8194979..9a691a95febc3 100644 --- a/aten/src/TH/THTensorDimApply.h +++ b/aten/src/TH/THTensorDimApply.h @@ -61,16 +61,16 @@ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] = 0; \ \ TENSOR1##_data = THTensor_getStoragePtr(TENSOR1)->data()+(TENSOR1)->storage_offset(); \ - TENSOR1##_stride = (TENSOR1)->stride(DIMENSION); \ - TENSOR1##_size = TENSOR1->size(DIMENSION); \ + TENSOR1##_stride = THTensor_strideLegacyNoScalars((TENSOR1), DIMENSION); \ + TENSOR1##_size = THTensor_sizeLegacyNoScalars((TENSOR1), DIMENSION); \ \ TENSOR2##_data = THTensor_getStoragePtr(TENSOR2)->data()+(TENSOR2)->storage_offset(); \ - TENSOR2##_stride = (TENSOR2)->stride(DIMENSION); \ - TENSOR2##_size = TENSOR2->size(DIMENSION); \ + TENSOR2##_stride = THTensor_strideLegacyNoScalars((TENSOR2), DIMENSION); \ + TENSOR2##_size = THTensor_sizeLegacyNoScalars((TENSOR2), DIMENSION); \ \ TENSOR3##_data = THTensor_getStoragePtr(TENSOR3)->data()+(TENSOR3)->storage_offset(); \ - TENSOR3##_stride = (TENSOR3)->stride(DIMENSION); \ - TENSOR3##_size = TENSOR3->size(DIMENSION); \ + TENSOR3##_stride = THTensor_strideLegacyNoScalars((TENSOR3), DIMENSION); \ + TENSOR3##_size = THTensor_sizeLegacyNoScalars((TENSOR3), DIMENSION); \ \ while(!TH_TENSOR_DIM_APPLY_hasFinished) \ { \ @@ -92,11 +92,11 @@ } \ \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]++; \ - TENSOR1##_data += TENSOR1->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR2##_data += TENSOR2->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR3##_data += TENSOR3->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR1##_data += THTensor_strideLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i); \ + TENSOR2##_data += THTensor_strideLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i); \ + TENSOR3##_data += THTensor_strideLegacyNoScalars(TENSOR3, TH_TENSOR_DIM_APPLY_i); \ \ - if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == TENSOR1->size(TH_TENSOR_DIM_APPLY_i)) \ + if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == THTensor_sizeLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i)) \ { \ if(TH_TENSOR_DIM_APPLY_i == THTensor_nDimensionLegacyNoScalars(TENSOR1)-1) \ { \ @@ -105,9 +105,9 @@ } \ else \ { \ - TENSOR1##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR1->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR2##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR2->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR3##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR3->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR1##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i); \ + TENSOR2##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i); \ + TENSOR3##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR3, TH_TENSOR_DIM_APPLY_i); \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] = 0; \ } \ } \ @@ -155,7 +155,7 @@ { \ if(TH_TENSOR_DIM_APPLY_i == DIMENSION) \ continue; \ - if(TENSOR1->size(TH_TENSOR_DIM_APPLY_i) != TENSOR2->size(TH_TENSOR_DIM_APPLY_i)) { \ + if(THTensor_sizeLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i) != THTensor_sizeLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i)) { \ AT_ERROR("Expected ", #TENSOR1, " ", TENSOR1->sizes(), " and ", #TENSOR2, " ", TENSOR2->sizes(), " to have the same size in dimension ", DIMENSION); \ } \ } \ @@ -168,12 +168,12 @@ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] = 0; \ \ TENSOR1##_data = THTensor_getStoragePtr(TENSOR1)->data()+(TENSOR1)->storage_offset(); \ - TENSOR1##_stride = (TENSOR1)->stride(DIMENSION); \ - TENSOR1##_size = TENSOR1->size(DIMENSION); \ + TENSOR1##_stride = THTensor_strideLegacyNoScalars((TENSOR1), DIMENSION); \ + TENSOR1##_size = THTensor_sizeLegacyNoScalars(TENSOR1, DIMENSION); \ \ TENSOR2##_data = THTensor_getStoragePtr(TENSOR2)->data()+(TENSOR2)->storage_offset(); \ - TENSOR2##_stride = (TENSOR2)->stride(DIMENSION); \ - TENSOR2##_size = TENSOR2->size(DIMENSION); \ + TENSOR2##_stride = THTensor_strideLegacyNoScalars((TENSOR2), DIMENSION); \ + TENSOR2##_size = THTensor_sizeLegacyNoScalars(TENSOR2, DIMENSION); \ \ while(!TH_TENSOR_DIM_APPLY_hasFinished) \ { \ @@ -195,10 +195,10 @@ } \ \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]++; \ - TENSOR1##_data += TENSOR1->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR2##_data += TENSOR2->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR1##_data += THTensor_strideLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i); \ + TENSOR2##_data += THTensor_strideLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i); \ \ - if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == TENSOR1->size(TH_TENSOR_DIM_APPLY_i)) \ + if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == THTensor_sizeLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i)) \ { \ if(TH_TENSOR_DIM_APPLY_i == THTensor_nDimensionLegacyNoScalars(TENSOR1)-1) \ { \ @@ -207,8 +207,8 @@ } \ else \ { \ - TENSOR1##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR1->stride(TH_TENSOR_DIM_APPLY_i); \ - TENSOR2##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR2->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR1##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i); \ + TENSOR2##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i); \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] = 0; \ } \ } \ @@ -270,8 +270,8 @@ THError("invalid dimension"); \ \ TENSOR##_data = THTensor_getStoragePtr(TENSOR)->data()+(TENSOR)->storage_offset(); \ - TENSOR##_stride = (TENSOR)->stride(DIMENSION); \ - TENSOR##_size = TENSOR->size(DIMENSION); \ + TENSOR##_stride = THTensor_strideLegacyNoScalars((TENSOR), DIMENSION); \ + TENSOR##_size = THTensor_sizeLegacyNoScalars(TENSOR, DIMENSION); \ /* Counter stores the indices into the Tensor at any time */ \ TH_TENSOR_DIM_APPLY_counter = (int64_t*)THAlloc(sizeof(int64_t)*(THTensor_nDimensionLegacyAll(TENSOR))); \ for(TH_TENSOR_DIM_APPLY_i = 0; TH_TENSOR_DIM_APPLY_i < THTensor_nDimensionLegacyAll(TENSOR); TH_TENSOR_DIM_APPLY_i++) \ @@ -302,9 +302,9 @@ \ /* Bump the counter at this index, update the pointer */ \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]++; \ - TENSOR##_data += TENSOR->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR##_data += THTensor_strideLegacyNoScalars(TENSOR, TH_TENSOR_DIM_APPLY_i); \ \ - if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == TENSOR->size(TH_TENSOR_DIM_APPLY_i)) \ + if(TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] == THTensor_sizeLegacyNoScalars(TENSOR, TH_TENSOR_DIM_APPLY_i)) \ { \ /* Handled TENSOR_size(dim) iterations for DIM_APPLY_i. If this is the last dimension, exit */ \ if(TH_TENSOR_DIM_APPLY_i == THTensor_nDimensionLegacyAll(TENSOR)-1) \ @@ -315,7 +315,7 @@ else \ { \ /* Reset the counter, and the pointer to the beginning of the storage for this combination of indices */ \ - TENSOR##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*TENSOR->stride(TH_TENSOR_DIM_APPLY_i); \ + TENSOR##_data -= TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i]*THTensor_strideLegacyNoScalars(TENSOR, TH_TENSOR_DIM_APPLY_i); \ TH_TENSOR_DIM_APPLY_counter[TH_TENSOR_DIM_APPLY_i] = 0; \ } \ } \ diff --git a/aten/src/TH/generic/THTensor.cpp b/aten/src/TH/generic/THTensor.cpp index e68c60a9455c4..a04e30b0dbe7c 100644 --- a/aten/src/TH/generic/THTensor.cpp +++ b/aten/src/TH/generic/THTensor.cpp @@ -439,7 +439,7 @@ void THTensor_(unfold)(THTensor *self, THTensor *src, int dimension, int64_t siz src = self; THArgCheck((dimension >= 0) && (dimension < THTensor_nDimensionLegacyNoScalars(src)), 2, "out of range"); - THArgCheck(size <= src->size(dimension), 3, "out of range"); + THArgCheck(size <= THTensor_sizeLegacyNoScalars(src, dimension), 3, "out of range"); THArgCheck(step > 0, 4, "invalid step"); THTensor_(set)(self, src); @@ -563,7 +563,7 @@ int THTensor_(isTransposed)(const THTensor *self) int64_t size_max_stride = 1; int64_t z = 1; int d; - for (d = 0; d < THTensor_nDimensionLegacyAll(self); ++d) { + for (d = 0; d < self->dim(); ++d) { if (self->stride(d) == 0 && self->size(d) != 1) return 0; if (self->stride(d) > max_stride) { @@ -596,20 +596,6 @@ int THTensor_(isContiguous)(const THTensor *self) return 1; } -int THTensor_(isSize)(const THTensor *self, const THLongStorage *dims) -{ - int d; - if (THTensor_nDimensionLegacyAll(self) != dims->size) - return 0; - - for(d = 0; d < THTensor_nDimensionLegacyAll(self); ++d) - { - if(self->size(d) != THLongStorage_data(dims)[d]) - return 0; - } - return 1; -} - int THTensor_(isSameSizeAs)(const THTensor *self, const THTensor* src) { int d; @@ -770,15 +756,15 @@ void THTensor_(resizeNd)(THTensor *self, int nDimension, int64_t *size, int64_t void THTensor_(set1d)(THTensor *tensor, int64_t x0, real value) { THArgCheck(THTensor_nDimensionLegacyNoScalars(tensor) == 1, 1, "tensor must have one dimension"); - THArgCheck( (x0 >= 0) && (x0 < tensor->size(0)), 2, "out of range"); - THStorage_(set)(THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*tensor->stride(0), value); + THArgCheck( (x0 >= 0) && (x0 < THTensor_sizeLegacyNoScalars(tensor, 0)), 2, "out of range"); + THStorage_(set)(THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*THTensor_strideLegacyNoScalars(tensor, 0), value); } real THTensor_(get1d)(const THTensor *tensor, int64_t x0) { THArgCheck(THTensor_nDimensionLegacyNoScalars(tensor) == 1, 1, "tensor must have one dimension"); - THArgCheck( (x0 >= 0) && (x0 < tensor->size(0)), 2, "out of range"); - return THStorage_(get)(THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*tensor->stride(0)); + THArgCheck( (x0 >= 0) && (x0 < THTensor_sizeLegacyNoScalars(tensor, 0)), 2, "out of range"); + return THStorage_(get)(THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*THTensor_strideLegacyNoScalars(tensor, 0)); } void THTensor_(set2d)(THTensor *tensor, int64_t x0, int64_t x1, real value) diff --git a/aten/src/TH/generic/THTensor.h b/aten/src/TH/generic/THTensor.h index 126fc504dee48..664fdd5a89f26 100644 --- a/aten/src/TH/generic/THTensor.h +++ b/aten/src/TH/generic/THTensor.h @@ -113,7 +113,6 @@ TH_API void THTensor_(unsqueeze1d)(THTensor *self, THTensor *src, int dimension_ TH_API int THTensor_(isContiguous)(const THTensor *self); TH_API int THTensor_(isSameSizeAs)(const THTensor *self, const THTensor *src); TH_API int THTensor_(isSetTo)(const THTensor *self, const THTensor *src); -TH_API int THTensor_(isSize)(const THTensor *self, const THLongStorage *dims); TH_API ptrdiff_t THTensor_(nElement)(const THTensor *self); TH_API void THTensor_(retain)(THTensor *self); diff --git a/aten/src/TH/generic/THTensorApply.hpp b/aten/src/TH/generic/THTensorApply.hpp index dd31314cb41ea..47fdaaf0df537 100644 --- a/aten/src/TH/generic/THTensorApply.hpp +++ b/aten/src/TH/generic/THTensorApply.hpp @@ -121,14 +121,14 @@ int shape_check_flag = 0; \ for(TH_TENSOR_DIM_APPLY_i = 0; TH_TENSOR_DIM_APPLY_i < THTensor_nDimensionLegacyAll(TENSOR1); TH_TENSOR_DIM_APPLY_i++) \ { \ - int64_t TENSOR3##_dim_size = TENSOR3->size(TH_TENSOR_DIM_APPLY_i); \ + int64_t TENSOR3##_dim_size = THTensor_sizeLegacyNoScalars(TENSOR3, TH_TENSOR_DIM_APPLY_i); \ if (TH_TENSOR_DIM_APPLY_i != DIMENSION) { \ - if (TENSOR3##_dim_size > TENSOR1->size(TH_TENSOR_DIM_APPLY_i)) { \ + if (TENSOR3##_dim_size > THTensor_sizeLegacyNoScalars(TENSOR1, TH_TENSOR_DIM_APPLY_i)) { \ shape_check_flag = 1; \ break; \ } \ } \ - if (TENSOR3##_dim_size > TENSOR2->size(TH_TENSOR_DIM_APPLY_i)) { \ + if (TENSOR3##_dim_size > THTensor_sizeLegacyNoScalars(TENSOR2, TH_TENSOR_DIM_APPLY_i)) { \ shape_check_flag = 1; \ break; \ } \ diff --git a/aten/src/TH/generic/THTensorEvenMoreMath.cpp b/aten/src/TH/generic/THTensorEvenMoreMath.cpp index 03946724dcadc..a2dfc3848ad39 100644 --- a/aten/src/TH/generic/THTensorEvenMoreMath.cpp +++ b/aten/src/TH/generic/THTensorEvenMoreMath.cpp @@ -132,8 +132,8 @@ void THTensor_(nonzero)(THLongTensor *subscript, THTensor *tensor) div = 1; for (dim = tensor->dim() - 1; dim >= 0; dim--) { - *(subscript_data + dim) = (i/div) % tensor->size(dim); - div *= tensor->size(dim); + *(subscript_data + dim) = (i/div) % THTensor_sizeLegacyNoScalars(tensor, dim); + div *= THTensor_sizeLegacyNoScalars(tensor, dim); } subscript_data += tensor->dim(); @@ -170,10 +170,11 @@ void THTensor_(indexSelect)(THTensor *tensor, THTensor *src, int dim, THLongTens { tensor_data = THTensor_(data)(tensor); src_data = THTensor_(data)(src); - ptrdiff_t rowsize = src->size(0) == 0 ? 1: THTensor_(nElement)(src) / src->size(0); + auto src_size0 = THTensor_sizeLegacyNoScalars(src, 0); + ptrdiff_t rowsize = src_size0 == 0 ? 1: THTensor_(nElement)(src) / src_size0; // check that the indices are within range - int64_t max = src->size(0) - 1 + TH_INDEX_BASE; + int64_t max = src_size0 - 1 + TH_INDEX_BASE; for (i=0; i max) { THLongTensor_free(index); @@ -422,7 +423,7 @@ void THTensor_(gather)(THTensor *tensor, THTensor *src, int dim, THLongTensor *i THArgCheck(THTensor_(nDimensionLegacyNoScalars)(src) == THTensor_(nDimensionLegacyNoScalars)(tensor), 2, "Input tensor must have same dimensions as output tensor"); - elems_per_row = THLongTensor_size(index, dim); + elems_per_row = THTensor_sizeLegacyNoScalars(index, dim); TH_TENSOR_DIM_APPLY3(real, tensor, real, src, int64_t, index, dim, TH_TENSOR_DIM_APPLY3_SIZE_EQ_EXCEPT_DIM, @@ -448,7 +449,7 @@ void THTensor_(scatter)(THTensor *tensor, int dim, THLongTensor *index, THTensor THArgCheck(THTensor_(nDimensionLegacyNoScalars)(src) == THTensor_(nDimensionLegacyNoScalars)(tensor), 4, "Input tensor must have same dimensions as output tensor"); - elems_per_row = THLongTensor_size(index, dim); + elems_per_row = THTensor_sizeLegacyNoScalars(index, dim); TH_TENSOR_DIM_APPLY3(real, tensor, real, src, int64_t, index, dim, TH_TENSOR_DIM_APPLY3_SIZE_SCATTER, @@ -474,7 +475,7 @@ void THTensor_(scatterAdd)(THTensor *tensor, int dim, THLongTensor *index, THTen THArgCheck(THTensor_(nDimensionLegacyNoScalars)(src) == THTensor_(nDimensionLegacyNoScalars)(tensor), 4, "Input tensor must have same dimensions as output tensor"); - elems_per_row = THLongTensor_size(index, dim); + elems_per_row = THTensor_sizeLegacyNoScalars(index, dim); TH_TENSOR_DIM_APPLY3(real, tensor, real, src, int64_t, index, dim, TH_TENSOR_DIM_APPLY3_SIZE_SCATTER, @@ -498,7 +499,7 @@ void THTensor_(scatterFill)(THTensor *tensor, int dim, THLongTensor *index, real THArgCheck(THLongTensor_nDimensionLegacyAll(index) == THTensor_(nDimensionLegacyAll)(tensor), 3, "Index tensor must have same dimensions as output tensor"); - elems_per_row = THLongTensor_size(index, dim); + elems_per_row = THTensor_sizeLegacyNoScalars(index, dim); TH_TENSOR_DIM_APPLY2(real, tensor, int64_t, index, dim, for (i = 0; i < elems_per_row; ++i) diff --git a/aten/src/TH/generic/THTensorFastGetSet.hpp b/aten/src/TH/generic/THTensorFastGetSet.hpp index 5ad3e13b237b8..b897aa5778a54 100644 --- a/aten/src/TH/generic/THTensorFastGetSet.hpp +++ b/aten/src/TH/generic/THTensorFastGetSet.hpp @@ -2,6 +2,10 @@ #define TH_GENERIC_FILE "generic/THTensorFastGetSet.hpp" #else +static inline real THTensor_(fastGetLegacy1dNoScalars)(THTensor *self, int64_t x0) { + return (THStorage_(data)(THTensor_getStoragePtr(self))+self->storage_offset())[(x0)*THTensor_strideLegacyNoScalars(self, 0)]; +} + static inline real THTensor_(fastGet1d)(THTensor *self, int64_t x0) { return (THStorage_(data)(THTensor_getStoragePtr(self))+self->storage_offset())[(x0)*self->stride(0)]; } diff --git a/aten/src/TH/generic/THTensorLapack.cpp b/aten/src/TH/generic/THTensorLapack.cpp index 73af3f8407080..e3855faff3a8c 100644 --- a/aten/src/TH/generic/THTensorLapack.cpp +++ b/aten/src/TH/generic/THTensorLapack.cpp @@ -373,7 +373,7 @@ void THTensor_(syev)(THTensor *re_, THTensor *rv_, THTensor *a, const char *jobz rv__ = THTensor_(cloneColumnMajor)(rv_, a); - n = rv__->size(0); + n = THTensor_sizeLegacyNoScalars(rv__, 0); lda = n; THTensor_(resize1d)(re_,n); @@ -688,7 +688,7 @@ void THTensor_(potri)(THTensor *ra_, THTensor *a, const char *uplo) ra__ = THTensor_(cloneColumnMajor)(ra_, a); - n = ra__->size(0); + n = THTensor_sizeLegacyNoScalars(ra__, 0); lda = n; /* Run inverse */ @@ -866,8 +866,8 @@ void THTensor_(orgqr)(THTensor *ra_, THTensor *a, THTensor *tau) THTensor *ra__ = NULL; ra__ = THTensor_(cloneColumnMajor)(ra_, a); - int m = ra__->size(0); - int k = tau->size(0); + int m = THTensor_sizeLegacyNoScalars(ra__, 0); + int k = THTensor_sizeLegacyNoScalars(tau, 0); int lda = m; /* Dry-run to query the suggested size of the workspace. */ @@ -921,7 +921,7 @@ void THTensor_(ormqr)(THTensor *ra_, THTensor *a, THTensor *tau, THTensor *c, co int m = c->size(0); int n = c->size(1); - int k = tau->size(0); + int k = THTensor_sizeLegacyNoScalars(tau, 0); int lda; if (*side == 'L') { diff --git a/aten/src/TH/generic/THTensorMath.cpp b/aten/src/TH/generic/THTensorMath.cpp index 24d9a7e8c4ea0..58c828120587b 100644 --- a/aten/src/TH/generic/THTensorMath.cpp +++ b/aten/src/TH/generic/THTensorMath.cpp @@ -815,10 +815,10 @@ void THTensor_(addmv)(THTensor *r_, real beta, THTensor *t, real alpha, THTensor THError("size mismatch, %s, %s", bm.str, bv.str); } - if(t->dim() != 1) + if(THTensor_nDimensionLegacyNoScalars(t) != 1) THError("vector expected, got t: %dD", t->dim()); - if(t->size(0) != mat->size(0)) { + if(THTensor_sizeLegacyNoScalars(t, 0) != mat->size(0)) { THDescBuff bt = THTensor_(sizeDesc)(t); THDescBuff bm = THTensor_(sizeDesc)(mat); THError("size mismatch, t: %s, mat: %s", bt.str, bm.str); @@ -830,6 +830,8 @@ void THTensor_(addmv)(THTensor *r_, real beta, THTensor *t, real alpha, THTensor THTensor_(copy)(r_, t); } + auto r_stride = THTensor_strideLegacyNoScalars(r_, 0); + // n == 1 || lda >= max(1, m) #define LDA_COND(M, N, LDA) ((N) == 1 || (LDA) >= THMax(1, (M))) @@ -838,14 +840,14 @@ void THTensor_(addmv)(THTensor *r_, real beta, THTensor *t, real alpha, THTensor THBlas_(gemv)('n', mat->size(0), mat->size(1), alpha, THTensor_(data)(mat), mat->stride(1), THTensor_(data)(vec), THTensor_strideLegacyNoScalars(vec, 0), - beta, THTensor_(data)(r_), r_->stride(0)); + beta, THTensor_(data)(r_), r_stride); } else if(mat->stride(1) == 1 && LDA_COND(mat->size(1), mat->size(0), mat->stride(0))) { THBlas_(gemv)('t', mat->size(1), mat->size(0), alpha, THTensor_(data)(mat), mat->stride(0), THTensor_(data)(vec), THTensor_strideLegacyNoScalars(vec, 0), - beta, THTensor_(data)(r_), r_->stride(0)); + beta, THTensor_(data)(r_), r_stride); } else { @@ -854,7 +856,7 @@ void THTensor_(addmv)(THTensor *r_, real beta, THTensor *t, real alpha, THTensor THBlas_(gemv)('t', mat->size(1), mat->size(0), alpha, THTensor_(data)(cmat), cmat->stride(0), THTensor_(data)(vec), THTensor_strideLegacyNoScalars(vec, 0), - beta, THTensor_(data)(r_), r_->stride(0)); + beta, THTensor_(data)(r_), r_stride); THTensor_(free)(cmat); } @@ -1099,7 +1101,7 @@ void THTensor_(addr)(THTensor *r_, real beta, THTensor *t, real alpha, THTensor THTensor_(data)(vec2), vec2_stride, THTensor_(data)(r_), r_->stride(1)); } - else if(r_->stride(1) == 1 && LDA_COND(vec2->size(0), vec1_size, r_->stride(0))) + else if(r_->stride(1) == 1 && LDA_COND(vec2_size, vec1_size, r_->stride(0))) { THBlas_(ger)(vec2_size, vec1_size, alpha, THTensor_(data)(vec2), vec2_stride, diff --git a/aten/src/TH/generic/THTensorMoreMath.cpp b/aten/src/TH/generic/THTensorMoreMath.cpp index fa8fb0558661e..e0dd8dc839af8 100644 --- a/aten/src/TH/generic/THTensorMoreMath.cpp +++ b/aten/src/TH/generic/THTensorMoreMath.cpp @@ -88,7 +88,7 @@ void THTensor_(max)(THTensor *values_, THLongTensor *indices_, THTensor *t, int THLongStorage_free(dim); // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { real theMax; real value; int64_t theIndex; @@ -121,7 +121,7 @@ void THTensor_(max)(THTensor *values_, THLongTensor *indices_, THTensor *t, int } THLongTensor_zero(indices_); - if(t->size(dimension) == 1) { + if(THTensor_sizeLegacyNoScalars(t, dimension) == 1) { if (!keepdim) { THTensor_(squeeze1d)(values_, values_, dimension); THLongTensor_squeeze1d(indices_, indices_, dimension); @@ -131,12 +131,12 @@ void THTensor_(max)(THTensor *values_, THLongTensor *indices_, THTensor *t, int THTensor *tempValues_ = THTensor_(newWithTensor)(values_); // tempValues_.expand_as(t) - THTensor_setSizeAtDim(tempValues_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(tempValues_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(tempValues_, dimension, 0); THLongTensor *tempIndices_ = THLongTensor_newWithTensor(indices_); // tempIndices_.expand_as(t) - THTensor_setSizeAtDim(tempIndices_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(tempIndices_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(tempIndices_, dimension, 0); TH_TENSOR_APPLY3_D(real, t, real, tempValues_, int64_t, tempIndices_, dimension, @@ -172,7 +172,7 @@ void THTensor_(min)(THTensor *values_, THLongTensor *indices_, THTensor *t, int THLongStorage_free(dim); // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { real theMax; real value; int64_t theIndex; @@ -205,7 +205,7 @@ void THTensor_(min)(THTensor *values_, THLongTensor *indices_, THTensor *t, int } THLongTensor_zero(indices_); - if(t->size(dimension) == 1) { + if(THTensor_sizeLegacyNoScalars(t, dimension) == 1) { if (!keepdim) { THTensor_(squeeze1d)(values_, values_, dimension); THLongTensor_squeeze1d(indices_, indices_, dimension); @@ -215,12 +215,12 @@ void THTensor_(min)(THTensor *values_, THLongTensor *indices_, THTensor *t, int THTensor *tempValues_ = THTensor_(newWithTensor)(values_); // tempValues_.expand_as(t) - THTensor_setSizeAtDim(tempValues_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(tempValues_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(tempValues_, dimension, 0); THLongTensor *tempIndices_ = THLongTensor_newWithTensor(indices_); // tempIndices_.expand_as(t) - THTensor_setSizeAtDim(tempIndices_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(tempIndices_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(tempIndices_, dimension, 0); TH_TENSOR_APPLY3_D(real, t, real, tempValues_, int64_t, tempIndices_, dimension, @@ -282,8 +282,8 @@ void THTensor_(sum)(THTensor *r_, THTensor *t, int dimension, int keepdim) real *t_data = tp+tBasicIndex; real *r__data = rp+iter; *r__data = 0; - for(j=0; j < t->size(dimension); ++j) { - *r__data += *(t_data + j*t->stride(dimension)); + for(j=0; j < THTensor_sizeLegacyNoScalars(t, dimension); ++j) { + *r__data += *(t_data + j*THTensor_strideLegacyNoScalars(t, dimension)); } } } else { @@ -295,7 +295,7 @@ void THTensor_(sum)(THTensor *r_, THTensor *t, int dimension, int keepdim) #endif if (serial_path) { // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { TH_TENSOR_DIM_APPLY2(real, t, real, r_, dimension, accreal sum = 0; int64_t i; @@ -306,7 +306,7 @@ void THTensor_(sum)(THTensor *r_, THTensor *t, int dimension, int keepdim) THTensor_(zero)(r_); THTensor *temp_ = THTensor_(newWithTensor)(r_); // r_.expand_as(t) - THTensor_setSizeAtDim(temp_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(temp_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(temp_, dimension, 0); TH_TENSOR_APPLY2(real, temp_, real, t, *temp__data = *temp__data + *t_data;); @@ -362,8 +362,8 @@ void THTensor_(prod)(THTensor *r_, THTensor *t, int dimension, int keepdim) real *t_data = tp+tBasicIndex; real *r__data = rp+iter; *r__data = 1; - for(j=0; j < t->size(dimension); ++j) { - *r__data *= *(t_data + j*t->stride(dimension)); + for(j=0; j < THTensor_sizeLegacyNoScalars(t, dimension); ++j) { + *r__data *= *(t_data + j*THTensor_strideLegacyNoScalars(t, dimension)); } } } else { @@ -376,7 +376,7 @@ void THTensor_(prod)(THTensor *r_, THTensor *t, int dimension, int keepdim) if(serial_path) { // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { TH_TENSOR_DIM_APPLY2(real, t, real, r_, dimension, accreal prod = 1; int64_t i; @@ -387,7 +387,7 @@ void THTensor_(prod)(THTensor *r_, THTensor *t, int dimension, int keepdim) THTensor_(fill)(r_, 1); THTensor *temp_ = THTensor_(newWithTensor)(r_); // r_.expand_as(t) - THTensor_setSizeAtDim(temp_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(temp_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(temp_, dimension, 0); TH_TENSOR_APPLY2(real, temp_, real, t, *temp__data = *temp__data * *t_data;); @@ -480,7 +480,7 @@ void THTensor_(cross)(THTensor *r_, THTensor *a, THTensor *b, int dimension) THError("inconsistent tensor dimension %dD, %dD", THTensor_(nDimensionLegacyNoScalars)(a), THTensor_(nDimensionLegacyNoScalars)(b)); - for(i = 0; i < THTensor_(nDimensionLegacyNoScalars)(a); i++) + for(i = 0; i < a->dim(); i++) { if(THTensor_(size)(a, i) != THTensor_(size)(b, i)) { THDescBuff ba = THTensor_(sizeDesc)(a); @@ -493,7 +493,7 @@ void THTensor_(cross)(THTensor *r_, THTensor *a, THTensor *b, int dimension) { for(i = 0; i < THTensor_(nDimensionLegacyNoScalars)(a); i++) { - if(THTensor_(size)(a, i) == 3) + if(THTensor_sizeLegacyNoScalars(a, i) == 3) { dimension = i; break; @@ -507,7 +507,7 @@ void THTensor_(cross)(THTensor *r_, THTensor *a, THTensor *b, int dimension) THArgCheck(dimension >= 0 && dimension < THTensor_(nDimensionLegacyNoScalars)(a), 3, "dimension %d out of range", dimension + TH_INDEX_BASE); - THArgCheck(THTensor_(size)(a, dimension) == 3, 3, "dimension %d does not have size 3", + THArgCheck(THTensor_sizeLegacyNoScalars(a, dimension) == 3, 3, "dimension %d does not have size 3", dimension + TH_INDEX_BASE); THTensor_(resizeAs)(r_, a); @@ -562,8 +562,8 @@ void THTensor_(diag)(THTensor *r_, THTensor *t, int k) if(THTensor_(nDimensionLegacyNoScalars)(t) == 1) { real *t_data = THTensor_(data)(t); - int64_t t_stride_0 = THTensor_(stride)(t, 0); - int64_t t_size = THTensor_(size)(t, 0); + int64_t t_stride_0 = THTensor_strideLegacyNoScalars(t, 0); + int64_t t_size = THTensor_sizeLegacyNoScalars(t, 0); int64_t sz = t_size + (k >= 0 ? k : -k); real *r__data; int64_t r__stride_0; @@ -1071,7 +1071,7 @@ void THTensor_(mode)(THTensor *values_, THLongTensor *indices_, THTensor *t, int THLongTensor_resize(indices_, dim, NULL); THLongStorage_free(dim); - t_size_dim = THTensor_(size)(t, dimension); + t_size_dim = THTensor_sizeLegacyNoScalars(t, dimension); temp_ = THTensor_(new)(); THTensor_(resize1d)(temp_, t_size_dim); @@ -1129,7 +1129,7 @@ void THTensor_(kthvalue)(THTensor *values_, THLongTensor *indices_, THTensor *t, int64_t t_size_dim; THArgCheck(dimension >= 0 && dimension < THTensor_(nDimensionLegacyAll)(t), 3, "dimension out of range"); - THArgCheck(k > 0 && k <= t->size(dimension), 2, "selected index out of range"); + THArgCheck(k > 0 && k <= THTensor_sizeLegacyNoScalars(t, dimension), 2, "selected index out of range"); int in_dims = THTensor_(nDimensionLegacyAll)(t); THTensor_(preserveReduceDimSemantics)(values_, in_dims, dimension, keepdim); @@ -1140,7 +1140,7 @@ void THTensor_(kthvalue)(THTensor *values_, THLongTensor *indices_, THTensor *t, THLongTensor_resize(indices_, dim, NULL); THLongStorage_free(dim); - t_size_dim = THTensor_(size)(t, dimension); + t_size_dim = THTensor_sizeLegacyNoScalars(t, dimension); temp_ = THTensor_(new)(); THTensor_(resize1d)(temp_, t_size_dim); @@ -1175,7 +1175,7 @@ void THTensor_(median)(THTensor *values_, THLongTensor *indices_, THTensor *t, i THArgCheck(dimension >= 0 && dimension < THTensor_(nDimensionLegacyAll)(t), 3, "dimension out of range"); - t_size_dim = THTensor_(size)(t, dimension); + t_size_dim = THTensor_sizeLegacyNoScalars(t, dimension); k = (t_size_dim-1) >> 1; /* take middle or one-before-middle element */ THTensor_(kthvalue)(values_, indices_, t, k+1, dimension, keepdim); @@ -1186,7 +1186,7 @@ void THTensor_(topk)(THTensor *rt_, THLongTensor *ri_, THTensor *t, int64_t k, i int numDims = THTensor_(nDimensionLegacyNoScalars)(t); THArgCheck(dim >= 0 && dim < numDims, 3, "dim not in range"); - int64_t sliceSize = THTensor_(size)(t, dim); + int64_t sliceSize = THTensor_sizeLegacyNoScalars(t, dim); THArgCheck(k >= 0 && k <= sliceSize, 2, "k not in range for dimension"); THTensor *tmpResults = THTensor_(new)(); @@ -1687,8 +1687,8 @@ void THTensor_(logicalAnd)(THTensor *r_, THTensor *t, int dimension, int keepdim real *t_data = tp+tBasicIndex; real *r__data = rp+iter; *r__data = 1; - for(j=0; j < t->size(dimension); ++j) { - *r__data = *r__data && *(t_data + j*t->stride(dimension)); + for(j=0; j < THTensor_sizeLegacyNoScalars(t, dimension); ++j) { + *r__data = *r__data && *(t_data + j*THTensor_strideLegacyNoScalars(t, dimension)); } } } else { @@ -1701,7 +1701,7 @@ void THTensor_(logicalAnd)(THTensor *r_, THTensor *t, int dimension, int keepdim if(serial_path) { // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { TH_TENSOR_DIM_APPLY2(real, t, real, r_, dimension, accreal prod = 1; int64_t i; @@ -1712,7 +1712,7 @@ void THTensor_(logicalAnd)(THTensor *r_, THTensor *t, int dimension, int keepdim THTensor_(fill)(r_, 1); THTensor *temp_ = THTensor_(newWithTensor)(r_); // r_.expand_as(t) - THTensor_setSizeAtDim(temp_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(temp_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(temp_, dimension, 0); TH_TENSOR_APPLY2(real, temp_, real, t, *temp__data = *temp__data && *t_data;); @@ -1767,8 +1767,8 @@ void THTensor_(logicalAny)(THTensor *r_, THTensor *t, int dimension, int keepdim real *t_data = tp+tBasicIndex; real *r__data = rp+iter; *r__data = 0; - for(j=0; j < t->size(dimension); ++j) { - *r__data = *r__data || *(t_data + j*t->stride(dimension)); + for(j=0; j < THTensor_sizeLegacyNoScalars(t, dimension); ++j) { + *r__data = *r__data || *(t_data + j*THTensor_strideLegacyNoScalars(t, dimension)); } } } else { @@ -1780,7 +1780,7 @@ void THTensor_(logicalAny)(THTensor *r_, THTensor *t, int dimension, int keepdim #endif if (serial_path) { // two implementations optimized for data locality - if (t->stride(dimension) == 1) { + if (THTensor_strideLegacyNoScalars(t, dimension) == 1) { TH_TENSOR_DIM_APPLY2(real, t, real, r_, dimension, accreal sum = 0; int64_t i; @@ -1791,7 +1791,7 @@ void THTensor_(logicalAny)(THTensor *r_, THTensor *t, int dimension, int keepdim THTensor_(zero)(r_); THTensor *temp_ = THTensor_(newWithTensor)(r_); // r_.expand_as(t) - THTensor_setSizeAtDim(temp_, dimension, t->size(dimension)); + THTensor_setSizeAtDim(temp_, dimension, THTensor_sizeLegacyNoScalars(t, dimension)); THTensor_setStrideAtDim(temp_, dimension, 0); TH_TENSOR_APPLY2(real, temp_, real, t, *temp__data = *temp__data || *t_data;); @@ -1876,7 +1876,7 @@ void THTensor_(mean)(THTensor *r_, THTensor *t, int dimension, int keepdim) dimension + TH_INDEX_BASE); THTensor_(sum)(r_, t, dimension, keepdim); - THTensor_(div)(r_, r_, t->size(dimension)); + THTensor_(div)(r_, r_, THTensor_sizeLegacyNoScalars(t, dimension)); } void THTensor_(std)(THTensor *r_, THTensor *t, int dimension, int biased, int keepdim) @@ -2054,7 +2054,7 @@ void THTensor_(renorm)(THTensor *res, THTensor *src, real value, int dimension, THTensor_(resizeAs)(res, src); - for (int64_t i = 0; i < src->size(dimension); i++) + for (int64_t i = 0; i < THTensor_sizeLegacyNoScalars(src, dimension); i++) { real norm = 0; real new_norm; @@ -2206,7 +2206,7 @@ void THTensor_(bhistc)(THTensor *hist, THTensor *tensor, int64_t nbins, real min real minval; real maxval; - THTensor_(resize2d)(hist, tensor->size(0), nbins); + THTensor_(resize2d)(hist, THTensor_sizeLegacyNoScalars(tensor, 0), nbins); THTensor_(zero)(hist); minval = minvalue; diff --git a/aten/src/THC/THCDeviceTensorUtils.cuh b/aten/src/THC/THCDeviceTensorUtils.cuh index 426bd1b6514c3..4a8da4a14c51c 100644 --- a/aten/src/THC/THCDeviceTensorUtils.cuh +++ b/aten/src/THC/THCDeviceTensorUtils.cuh @@ -58,8 +58,8 @@ toDeviceTensor(THCState* state, THCTensor* t) { IndexT strides[Dim]; for (int i = 0; i < Dim; ++i) { - int64_t size = THCTensor_size(state, t, i); - int64_t stride = THCTensor_stride(state, t, i); + int64_t size = THTensor_sizeLegacyNoScalars(t, i); + int64_t stride = THTensor_strideLegacyNoScalars(t, i); maxOffset += (size - 1) * stride; diff --git a/aten/src/THC/THCReduce.cuh b/aten/src/THC/THCReduce.cuh index 93adc5034f186..df1ad7c9aec3e 100644 --- a/aten/src/THC/THCReduce.cuh +++ b/aten/src/THC/THCReduce.cuh @@ -410,8 +410,8 @@ bool THC_reduceDim(THCState* state, int keepdim) { ptrdiff_t inElements = THCTensor_nElement(state, in); - int64_t reductionSize = THCTensor_size(state, in, dim); - int64_t reductionStride = THCTensor_stride(state, in, dim); + int64_t reductionSize = THTensor_sizeLegacyNoScalars(in, dim); + int64_t reductionStride = THTensor_strideLegacyNoScalars(in, dim); ptrdiff_t outElements = inElements / reductionSize; if (THCTensor_nDimensionLegacyAll(state, out) > MAX_CUTORCH_DIMS || diff --git a/aten/src/THC/THCTensor.cpp b/aten/src/THC/THCTensor.cpp index a8fb33c11a5bd..2ca851d9c2cad 100644 --- a/aten/src/THC/THCTensor.cpp +++ b/aten/src/THC/THCTensor.cpp @@ -22,10 +22,20 @@ int64_t THCTensor_size(THCState *state, const THCTensor *self, int dim) { return self->size(dim); } +int64_t THCTensor_sizeLegacyNoScalars(THCState *state, const THCTensor *self, int dim) { + return THTensor_sizeLegacyNoScalars(self, dim); +} + + int64_t THCTensor_stride(THCState *state, const THCTensor *self, int dim) { THArgCheck((dim >= 0) && (dim < self->dim()), 2, "out of range"); return self->stride(dim); } + +int64_t THCTensor_strideLegacyNoScalars(THCState *state, const THCTensor *self, int dim) { + return THTensor_strideLegacyNoScalars(self, dim); +} + THLongStorage *THCTensor_newSizeOf(THCState *state, THCTensor *self) { THLongStorage *size = THLongStorage_newWithSize(self->dim()); THLongStorage_rawCopy(size, THTensor_getSizePtr(self)); @@ -310,6 +320,9 @@ bool THCTensor_canUse32BitIndexMath(THCState* state, const THCTensor* t, ptrdiff if (elements >= max_elem) { return false; } + if (t->dim() == 0) { + return true; + } ptrdiff_t offset = 0; ptrdiff_t linearId = elements - 1; @@ -393,7 +406,7 @@ bool THCTensor_maybeOverlappingIndices(THCState* state, const THCTensor* t) { int dims = THCTensor_nDimensionLegacyAll(state, t); int nonSize1Dims = 0; for (int i = 0; i < dims; ++i) { - int64_t size = THCTensor_size(state, t, i); + int64_t size = THCTensor_sizeLegacyNoScalars(state, t, i); if (size > 1) { info[nonSize1Dims].size = size; diff --git a/aten/src/THC/THCTensor.hpp b/aten/src/THC/THCTensor.hpp index 3dbc7d0151cd5..2ed8cbf808bd1 100644 --- a/aten/src/THC/THCTensor.hpp +++ b/aten/src/THC/THCTensor.hpp @@ -16,7 +16,9 @@ THC_API int THCTensor_nDimensionLegacyNoScalars(THCState *state, const THCTensor THC_API int THCTensor_nDimensionLegacyAll(THCState *state, const THCTensor *self); THC_API int64_t THCTensor_size(THCState *state, const THCTensor *self, int dim); +THC_API int64_t THCTensor_sizeLegacyNoScalars(THCState *state, const THCTensor *self, int dim); THC_API int64_t THCTensor_stride(THCState *state, const THCTensor *self, int dim); +THC_API int64_t THCTensor_strideLegacyNoScalars(THCState *state, const THCTensor *self, int dim); THC_API THLongStorage *THCTensor_newSizeOf(THCState *state, THCTensor *self); THC_API THCTensor *THCTensor_new(THCState *state, at::ScalarType scalar_type); diff --git a/aten/src/THC/THCTensorMathReduce.cuh b/aten/src/THC/THCTensorMathReduce.cuh index b680595bc51e7..cc3c7cb499bea 100644 --- a/aten/src/THC/THCTensorMathReduce.cuh +++ b/aten/src/THC/THCTensorMathReduce.cuh @@ -300,13 +300,13 @@ __host__ void THCTensor_varOuterDim(THCState *state, TensorTypeK *tgt, TensorTyp // Treat all outer dimensions (i.e. dim < dimension) as one. unsigned num_orows = 1; for (int64_t dim = 0; dim < dimension; dim++) { - num_orows *= THCTensor_size(state, src, dim); + num_orows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } - unsigned row_size = THCTensor_size(state, src, dimension); + unsigned row_size = THCTensor_sizeLegacyNoScalars(state, src, dimension); // Treat all inner dimensions (i.e. dim > dimension) as one. unsigned num_irows = 1; for (unsigned dim = dimension + 1; dim < ndim; dim++) { - num_irows *= THCTensor_size(state, src, dim); + num_irows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } dim3 threads(min(512, num_irows)); @@ -446,9 +446,9 @@ __host__ void THCTensor_varInnermostDim(THCState *state, TensorTypeK *tgt, Tenso // Treat all outer dimensions as a single dimension. unsigned num_rows = 1; for (unsigned dim = 0; dim < ndim - 1; dim++) { - num_rows *= THCTensor_size(state, src, dim); + num_rows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } - unsigned row_size = THCTensor_size(state, src, ndim - 1); + unsigned row_size = THCTensor_sizeLegacyNoScalars(state, src, ndim - 1); // From limited testing, 16x32 seemed a good compromise for handling both long and short dimensions. dim3 threads(16, 32); @@ -518,12 +518,12 @@ THC_transformReduceOuterDimIndex(THCState *state, unsigned ndim = THCTensor_nDimensionLegacyAll(state, src); unsigned num_orows = 1; for (int64_t dim = 0; dim < rdim; dim++) { - num_orows *= THCTensor_size(state, src, dim); + num_orows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } - unsigned row_size = THCTensor_size(state, src, rdim); + unsigned row_size = THCTensor_sizeLegacyNoScalars(state, src, rdim); unsigned num_irows = 1; for (unsigned dim = rdim + 1; dim < ndim; dim++) { - num_irows *= THCTensor_size(state, src, dim); + num_irows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } dim3 threads(min(512, num_irows)); @@ -621,9 +621,9 @@ THC_transformReduceInnermostDimIndex(THCState *state, unsigned ndim = THCTensor_nDimensionLegacyAll(state, src); unsigned num_rows = 1; for (unsigned dim = 0; dim < ndim - 1; dim++) { - num_rows *= THCTensor_size(state, src, dim); + num_rows *= THCTensor_sizeLegacyNoScalars(state, src, dim); } - unsigned row_size = THCTensor_size(state, src, ndim - 1); + unsigned row_size = THCTensor_sizeLegacyNoScalars(state, src, ndim - 1); dim3 threads(16, 32); dim3 grid(min(1024, THCCeilDiv(num_rows, threads.y))); diff --git a/aten/src/THC/THCTensorSort.cu b/aten/src/THC/THCTensorSort.cu index 6a5ee6e6864d0..25ce459c48de7 100644 --- a/aten/src/THC/THCTensorSort.cu +++ b/aten/src/THC/THCTensorSort.cu @@ -8,7 +8,7 @@ void THCudaLongTensor_fillSliceWithIndex(THCState* state, ptrdiff_t inElements = THCudaLongTensor_nElement(state, t); if (inElements > 0) { - int64_t sliceSize = THCudaLongTensor_size(state, t, dim); + int64_t sliceSize = THCudaLongTensor_sizeLegacyNoScalars(state, t, dim); ptrdiff_t numSlices = inElements / sliceSize; dim3 grid; diff --git a/aten/src/THC/THCTensorTypeUtils.cuh b/aten/src/THC/THCTensorTypeUtils.cuh index 94c3da69d90bf..b5dc6c547ba50 100644 --- a/aten/src/THC/THCTensorTypeUtils.cuh +++ b/aten/src/THC/THCTensorTypeUtils.cuh @@ -62,8 +62,8 @@ getTensorInfo(THCState* state, TensorType* t) { int dims = THCTensor_nDimensionLegacyNoScalars(state, t); for (int i = 0; i < dims; ++i) { - sz[i] = THCTensor_size(state, t, i); - st[i] = THCTensor_stride(state, t, i); + sz[i] = THTensor_sizeLegacyNoScalars(t, i); + st[i] = THTensor_strideLegacyNoScalars(t, i); } return TensorInfo( diff --git a/aten/src/THC/generic/THCTensor.cpp b/aten/src/THC/generic/THCTensor.cpp index 940af6eb86ead..fdf80565bd413 100644 --- a/aten/src/THC/generic/THCTensor.cpp +++ b/aten/src/THC/generic/THCTensor.cpp @@ -523,20 +523,6 @@ int THCTensor_(isContiguous)(THCState *state, const THCTensor *self) return THCTensor_isContiguous(state, self); } -int THCTensor_(isSize)(THCState *state, const THCTensor *self, const THLongStorage *dims) -{ - int d; - if (self->dim() != dims->size) - return 0; - - for (d = 0; d < self->dim(); ++d) - { - if (self->size(d) != THLongStorage_data(dims)[d]) - return 0; - } - return 1; -} - int THCTensor_(isSetTo)(THCState *state, const THCTensor *self, const THCTensor *src) { if (THTensor_getStoragePtr(self) == THTensor_getStoragePtr(src) && @@ -606,14 +592,14 @@ void THCTensor_(set1d)(THCState *state, THCTensor *tensor, int64_t x0, real valu { THArgCheck(THTensor_nDimensionLegacyNoScalars(tensor) == 1, 1, "tensor must have one dimension"); THArgCheck( (x0 >= 0) && (x0 < THTensor_sizeLegacyNoScalars(tensor, 0)), 2, "out of range"); - THCStorage_(set)(state, THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*tensor->stride(0), value); + THCStorage_(set)(state, THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*THTensor_strideLegacyNoScalars(tensor, 0), value); } real THCTensor_(get1d)(THCState *state, const THCTensor *tensor, int64_t x0) { THArgCheck(THTensor_nDimensionLegacyNoScalars(tensor) == 1, 1, "tensor must have one dimension"); THArgCheck( (x0 >= 0) && (x0 < THTensor_sizeLegacyNoScalars(tensor, 0)), 2, "out of range"); - return THCStorage_(get)(state, THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*tensor->stride(0)); + return THCStorage_(get)(state, THTensor_getStoragePtr(tensor), tensor->storage_offset()+x0*THTensor_strideLegacyNoScalars(tensor, 0)); } void THCTensor_(set2d)(THCState *state, THCTensor *tensor, int64_t x0, int64_t x1, real value) diff --git a/aten/src/THC/generic/THCTensor.h b/aten/src/THC/generic/THCTensor.h index 2ee1bf11a4be4..3f6b94eb4ba52 100644 --- a/aten/src/THC/generic/THCTensor.h +++ b/aten/src/THC/generic/THCTensor.h @@ -115,7 +115,6 @@ THC_API void THCTensor_(unsqueeze1d)(THCState *state, THCTensor *self, THCTensor THC_API int THCTensor_(isContiguous)(THCState *state, const THCTensor *self); THC_API int THCTensor_(isSameSizeAs)(THCState *state, const THCTensor *self, const THCTensor *src); THC_API int THCTensor_(isSetTo)(THCState *state, const THCTensor *self, const THCTensor *src); -THC_API int THCTensor_(isSize)(THCState *state, const THCTensor *self, const THLongStorage *dims); THC_API ptrdiff_t THCTensor_(nElement)(THCState *state, const THCTensor *self); THC_API void THCTensor_(retain)(THCState *state, THCTensor *self); diff --git a/aten/src/THC/generic/THCTensorIndex.cu b/aten/src/THC/generic/THCTensorIndex.cu index 82f56f9946e47..111ce61e502f7 100644 --- a/aten/src/THC/generic/THCTensorIndex.cu +++ b/aten/src/THC/generic/THCTensorIndex.cu @@ -19,14 +19,14 @@ static ptrdiff_t THCTensor_(getSliceSize)(THCState *state, THCTensor *dst, ptrdiff_t dstSliceSize = 1; for (int d = 0; d < dstDims; d++) { if (d != dim) { - dstSliceSize *= dst->size(d); + dstSliceSize *= THTensor_sizeLegacyNoScalars(dst, d); } } if (src == nullptr) return dstSliceSize; THArgCheck(dim < srcDims, 3, "Indexing dim is out of bounds"); - THArgCheck(THCudaLongTensor_nElement(state, index) == src->size(dim), 4, + THArgCheck(THCudaLongTensor_nElement(state, index) == THTensor_sizeLegacyNoScalars(src, dim), 4, "length of src.size[dim] is not equal to length of indices"); ptrdiff_t srcSliceSize = 1; @@ -36,8 +36,8 @@ static ptrdiff_t THCTensor_(getSliceSize)(THCState *state, THCTensor *dst, for (int d = 0; d < srcDims; d++) { if (d != dim) { - srcSliceSize *= src->size(d); - if (!mismatch && dst->size(d) != src->size(d)) mismatch = true; + srcSliceSize *= THTensor_sizeLegacyNoScalars(src, d); + if (!mismatch && THTensor_sizeLegacyNoScalars(dst, d) != THTensor_sizeLegacyNoScalars(src, d)) mismatch = true; } } @@ -111,7 +111,7 @@ void THCTensor_(indexCopy)(THCState *state, THCTensor *dst, int dim, THCudaLongT // of the tensor `indices`. ptrdiff_t sliceSize = THCTensor_(getSliceSize)(state, dst, dim, indices, src); ptrdiff_t srcTotalSize = THCTensor_(nElement)(state, src); - int64_t dstCopyDimSize = THCTensor_(size)(state, dst, dim); + int64_t dstCopyDimSize = THCTensor_(sizeLegacyNoScalars)(state, dst, dim); ptrdiff_t numIndices = THCudaLongTensor_nElement(state, indices); if (sliceSize == 0) { @@ -300,7 +300,7 @@ void THCTensor_(indexAdd)(THCState *state, THCTensor *dst, int dim, THCudaLongTe // of the tensor `indices`. ptrdiff_t sliceSize = THCTensor_(getSliceSize)(state, dst, dim, indices, src); ptrdiff_t srcTotalSize = THCTensor_(nElement)(state, src); - int64_t dstAddDimSize = THCTensor_(size)(state, dst, dim); + int64_t dstAddDimSize = THCTensor_(sizeLegacyNoScalars)(state, dst, dim); ptrdiff_t numIndices = THCudaLongTensor_nElement(state, indices); if (sliceSize == 0) { @@ -422,7 +422,7 @@ void THCTensor_(indexFill)(THCState *state, THCTensor *dst, int dim, THCudaLongT ptrdiff_t sliceSize = THCTensor_(getSliceSize)(state, dst, dim, indices, nullptr); ptrdiff_t dstTotalSize = THCTensor_(nElement)(state, dst); - int64_t dstFillDimSize = THCTensor_(size)(state, dst, dim); + int64_t dstFillDimSize = THCTensor_(sizeLegacyNoScalars)(state, dst, dim); ptrdiff_t numIndices = THCudaLongTensor_nElement(state, indices); if (sliceSize == 0) { @@ -554,7 +554,7 @@ void THCTensor_(indexSelect)(THCState *state, THCTensor *dst, THCTensor *src, in // total size of the tensor ignoring dimension `dim`; // -the number of indices we are choosing, which is the total size // of the tensor `indices`. - int64_t srcSelectDimSize = THCTensor_(size)(state, src, dim); + int64_t srcSelectDimSize = THCTensor_(sizeLegacyNoScalars)(state, src, dim); ptrdiff_t sliceSize = dstTotalSize / numIndices; int mpc = THCState_getCurrentDeviceProperties(state)->multiProcessorCount; diff --git a/aten/src/THC/generic/THCTensorMath.cu b/aten/src/THC/generic/THCTensorMath.cu index cc1a8c9ba57e4..7fa0dbe1e190c 100644 --- a/aten/src/THC/generic/THCTensorMath.cu +++ b/aten/src/THC/generic/THCTensorMath.cu @@ -314,9 +314,9 @@ void THCTensor_(nonzero)(THCState* state, THCudaLongTensor *tensor, strided_tensor.begin(), strided_tensor.end(), stride_dim.begin(), - idx_functor(div, self->size(dim)) + idx_functor(div, THTensor_sizeLegacyNoScalars(self, dim)) ); - div *= self->size(dim); + div *= THTensor_sizeLegacyNoScalars(self, dim); } THCudaLongTensor_resize2d(state, tensor, num_nonzeros, num_dim); @@ -349,7 +349,7 @@ void THCTensor_(diag)(THCState *state, THCTensor *self_, THCTensor *src_, int64_ } else { ptrdiff_t totalElements = THCTensor_(nElement)(state, src_); ptrdiff_t size = (k > 0) ? totalElements + k : totalElements - k; - int64_t strideSrc = THCTensor_(stride)(state, src_, 0); + int64_t strideSrc = THTensor_strideLegacyNoScalars(src_, 0); THCTensor_(resize2d)(state, self_, size, size); THCTensor_(zero)(state, self_); if (size > 0) { diff --git a/aten/src/THC/generic/THCTensorMathBlas.cu b/aten/src/THC/generic/THCTensorMathBlas.cu index 591780b04edf7..0ab8165bf9443 100644 --- a/aten/src/THC/generic/THCTensorMathBlas.cu +++ b/aten/src/THC/generic/THCTensorMathBlas.cu @@ -63,7 +63,7 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real if(t->dim() != 1) THError("size mismatch"); - if(t->size(0) != mat->size(0)) + if(THTensor_sizeLegacyNoScalars(t, 0) != mat->size(0)) THError("size mismatch"); #if defined(THC_REAL_IS_FLOAT) || defined(THC_REAL_IS_DOUBLE) @@ -73,18 +73,20 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real THCTensor_(copy)(state, r_, t); } + auto r_stride = THTensor_strideLegacyNoScalars(r_, 0); + if(mat->stride(0) == 1) { #ifdef THC_REAL_IS_FLOAT THCudaBlas_Sgemv(state, 'n', mat->size(0), mat->size(1), alpha, THCTensor_(data)(state, mat), mat->stride(1), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #elif defined(THC_REAL_IS_DOUBLE) THCudaBlas_Dgemv(state, 'n', mat->size(0), mat->size(1), alpha, THCTensor_(data)(state, mat), mat->stride(1), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #endif } else if(mat->stride(1) == 1) @@ -93,12 +95,12 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real THCudaBlas_Sgemv(state, 't', mat->size(1), mat->size(0), alpha, THCTensor_(data)(state, mat), mat->stride(0), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #elif defined(THC_REAL_IS_DOUBLE) THCudaBlas_Dgemv(state, 't', mat->size(1), mat->size(0), alpha, THCTensor_(data)(state, mat), mat->stride(0), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #endif } else @@ -109,12 +111,12 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real THCudaBlas_Sgemv(state, 't', mat->size(1), mat->size(0), alpha, THCTensor_(data)(state, cmat), cmat->stride(0), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #elif defined(THC_REAL_IS_DOUBLE) THCudaBlas_Dgemv(state, 't', mat->size(1), mat->size(0), alpha, THCTensor_(data)(state, cmat), cmat->stride(0), THCTensor_(data)(state, vec), vec_stride, - beta, THCTensor_(data)(state, r_), r_->stride(0)); + beta, THCTensor_(data)(state, r_), r_stride); #endif THCTensor_(free)(state, cmat); @@ -122,7 +124,7 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real // In cublasSgemv, cublasDgemv (x,0).mv(0) does not // handle beta, whereas cublasSgemm, cublasDgemm do for case where (x,0).mm(0,y). - if (vec->size(0) == 0 && mat->size(0) != 0) { + if (THTensor_sizeLegacyNoScalars(vec, 0) == 0 && mat->size(0) != 0) { if(THCNumerics::eq(beta, ScalarConvert::to(0))) { THCTensor_(zero)(state, r_); } else if(THCNumerics::ne(beta, ScalarConvert::to(1))) { @@ -136,12 +138,12 @@ THCTensor_(addmv)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real THCTensor_(resize2d)(state, vecAsMatrix, vec_size, 1); THCTensor *tAsMatrix = THCTensor_(newWithTensor)(state, t); - THCTensor_(resize2d)(state, tAsMatrix, tAsMatrix->size(0), 1); + THCTensor_(resize2d)(state, tAsMatrix, THTensor_sizeLegacyNoScalars(tAsMatrix, 0), 1); THCTensor_(addmm)(state, r_, beta, tAsMatrix, alpha, mat, vecAsMatrix); // r_ will have answer as matrix, need to return a vector - THCTensor_(resize1d)(state, r_, r_->size(0)); + THCTensor_(resize1d)(state, r_, THTensor_sizeLegacyNoScalars(r_, 0)); THCTensor_(free)(state, vecAsMatrix); THCTensor_(free)(state, tAsMatrix); #endif @@ -192,7 +194,7 @@ THCTensor_(addr)(THCState *state, THCTensor *r_, real beta, THCTensor *t, real a THCTensor_(data)(state, vec2), vec2_stride, THCTensor_(data)(state, r_), r_->stride(1)); #elif defined(THC_REAL_IS_DOUBLE) - THCudaBlas_Dger(state, vec1->size(0), vec2_size, + THCudaBlas_Dger(state, vec1_size, vec2_size, alpha, THCTensor_(data)(state, vec1), vec1_stride, THCTensor_(data)(state, vec2), vec2_stride, THCTensor_(data)(state, r_), r_->stride(1)); diff --git a/aten/src/THC/generic/THCTensorMathMagma.cu b/aten/src/THC/generic/THCTensorMathMagma.cu index dd2b2777d3552..aee04a8e22a4e 100644 --- a/aten/src/THC/generic/THCTensorMathMagma.cu +++ b/aten/src/THC/generic/THCTensorMathMagma.cu @@ -185,7 +185,7 @@ THC_API void THCTensor_(gels)(THCState *state, THCTensor *rb_, THCTensor *ra_, T THC_API void THCTensor_(syev)(THCState *state, THCTensor *re_, THCTensor *rv_, THCTensor *a, const char *jobzs, const char *uplos) { #ifdef USE_MAGMA - int64_t n = a->size(0); + int64_t n = THTensor_sizeLegacyNoScalars(a, 0); int64_t lda = n; magma_uplo_t uplo = uplos[0] == 'U' ? MagmaUpper : MagmaLower; diff --git a/aten/src/THC/generic/THCTensorMathPointwise.cu b/aten/src/THC/generic/THCTensorMathPointwise.cu index 46ddc50ffd693..a239da6635a15 100644 --- a/aten/src/THC/generic/THCTensorMathPointwise.cu +++ b/aten/src/THC/generic/THCTensorMathPointwise.cu @@ -114,9 +114,9 @@ THCTensor_(cross)(THCState *state, THCTensor *self, THCTensor *x, THCTensor *y, THCAssertSameGPU(THCTensor_(checkGPU)(state, 3, self, x, y)); int i; - int nd = THCTensor_(nDimensionLegacyNoScalars)(state, x); + int nd = x->dim(); ptrdiff_t nelem = THCTensor_(nElement)(state, x); - THArgCheck(nd == THCTensor_(nDimensionLegacyNoScalars)(state, y), 1, "tensors must have same number of dimensions"); + THArgCheck(nd == y->dim(), 1, "tensors must have same number of dimensions"); for (i = 0; i < nd; i++) { THArgCheck(THCTensor_(size)(state, x, i) == THCTensor_(size)(state, y, i), 1, "dimension %i of x and y does not match", i); if (dimension < 0 && THCTensor_(size)(state, x, i) == 3) { diff --git a/aten/src/THC/generic/THCTensorMathReduce.cu b/aten/src/THC/generic/THCTensorMathReduce.cu index 4300cc62ae4b4..614ce49d58274 100644 --- a/aten/src/THC/generic/THCTensorMathReduce.cu +++ b/aten/src/THC/generic/THCTensorMathReduce.cu @@ -68,8 +68,8 @@ THCTensor_(renorm)(THCState *state, THCTensor* self, THCTensor* src, real value, THArgCheck(THCTensor_(nDimensionLegacyNoScalars)(state, src) > 1, 1, "need at least 2 dimensions"); if (numel > 0) { - ptrdiff_t size = numel / data->size(0); - dim3 grid(data->size(0)); + ptrdiff_t size = numel / THTensor_sizeLegacyNoScalars(data, 0); + dim3 grid( THTensor_sizeLegacyNoScalars(data, 0)); dim3 threads(32); THCTensor_kernel_renorm diff --git a/aten/src/THC/generic/THCTensorMathScan.cu b/aten/src/THC/generic/THCTensorMathScan.cu index 3bb1c0bf1233a..708cb2ed4d63e 100644 --- a/aten/src/THC/generic/THCTensorMathScan.cu +++ b/aten/src/THC/generic/THCTensorMathScan.cu @@ -32,13 +32,13 @@ __host__ void THCTensor_(scanOuterDim)(THCState *state, THCTensor *tgt, // Treat all outer dimensions (i.e. dim < dimension) as one. unsigned num_orows = 1; for (int dim = 0; dim < dimension; dim++) { - num_orows *= THCTensor_(size)(state, src, dim); + num_orows *= THCTensor_(sizeLegacyNoScalars)(state, src, dim); } - unsigned row_size = THCTensor_(size)(state, src, dimension); + unsigned row_size = THCTensor_(sizeLegacyNoScalars)(state, src, dimension); // Treat all inner dimensions (i.e. dim > dimension) as one. unsigned num_irows = 1; for (unsigned dim = dimension + 1; dim < ndim; dim++) { - num_irows *= THCTensor_(size)(state, src, dim); + num_irows *= THCTensor_(sizeLegacyNoScalars)(state, src, dim); } dim3 threads(min(512, num_irows)); @@ -61,9 +61,9 @@ __host__ void THCTensor_(scanInnermostDim)(THCState *state, THCTensor *tgt, // Treat all outer dimensions as a single dimension. unsigned num_rows = 1; for (unsigned dim = 0; dim < ndim - 1; dim++) { - num_rows *= THCTensor_(size)(state, src, dim); + num_rows *= THCTensor_(sizeLegacyNoScalars)(state, src, dim); } - unsigned row_size = THCTensor_(size)(state, src, ndim - 1); + unsigned row_size = THCTensor_(sizeLegacyNoScalars)(state, src, ndim - 1); dim3 threads(16, 32); dim3 grid(min(1024, THCCeilDiv(num_rows, threads.y))); diff --git a/aten/src/THC/generic/THCTensorMode.cu b/aten/src/THC/generic/THCTensorMode.cu index a3c1cc8d067d7..1903995994d5f 100644 --- a/aten/src/THC/generic/THCTensorMode.cu +++ b/aten/src/THC/generic/THCTensorMode.cu @@ -17,10 +17,10 @@ THC_API void THCTensor_(calculateMode)(THCState *state, // calculations to get an offset real *data = THCTensor_(data)(state, input); for (int i = 0; i < THLongStorage_size(position); ++i) { - data += THLongStorage_data(position)[i] * THCTensor_(stride)(state, input, i); + data += THLongStorage_data(position)[i] * THTensor_strideLegacyNoScalars(input, i); } - int64_t nElement = THCTensor_(size)(state, input, THCTensor_(nDimensionLegacyAll)(state, input) - 1); + int64_t nElement = THCTensor_(sizeLegacyNoScalars)(state, input, THCTensor_(nDimensionLegacyAll)(state, input) - 1); THCThrustAllocator thrustAlloc(state); // Wrap input data, sortBuffer, in Thrust device vectors @@ -121,8 +121,8 @@ THC_API void THCTensor_(calculateMode)(THCState *state, for (int i = 0; i < THLongStorage_size(position); ++i) { int64_t pos = THLongStorage_data(position)[i]; - valuesOffset += THCTensor_(stride)(state, values, i) * pos; - indicesOffset += THCudaLongTensor_stride(state, indices, i) * pos; + valuesOffset += THTensor_strideLegacyNoScalars(values, i) * pos; + indicesOffset += THTensor_strideLegacyNoScalars(indices, i) * pos; } THCStorage_(set)(state, THCTensor_(storage)(state, values), valuesOffset, mode); THCudaLongStorage_set(state, THCudaLongTensor_storage(state, indices), indicesOffset, index); @@ -145,7 +145,7 @@ THC_API void THCTensor_(dimApplyMode)(THCState *state, THCTensor_(calculateMode)(state, values, indices, input, sortBuffer, dimension, position); } else { // Loop through the values and recurse - for (int i = 0; i < THCTensor_(size)(state, input, curDim); ++i) { + for (int i = 0; i < THCTensor_(sizeLegacyNoScalars)(state, input, curDim); ++i) { THLongStorage_data(position)[curDim] = i; THCTensor_(dimApplyMode)(state, values, indices, input, sortBuffer, dimension, position, curDim + 1); } @@ -175,7 +175,7 @@ THC_API void THCTensor_(mode)(THCState *state, ndim = THCTensor_(nDimensionLegacyAll)(state, input); THArgCheck(dimension >= 0 && dimension < ndim, 4, "Dimension of out bounds"); - sliceSize = THCTensor_(size)(state, input, dimension); + sliceSize = THCTensor_(sizeLegacyNoScalars)(state, input, dimension); slices = THCTensor_(nElement)(state, input) / sliceSize; // Resize output value, index Tensors to appropriate sizes (i.e. the same as diff --git a/aten/src/THC/generic/THCTensorRandom.cu b/aten/src/THC/generic/THCTensorRandom.cu index c03daad934beb..554b68fe9ea81 100644 --- a/aten/src/THC/generic/THCTensorRandom.cu +++ b/aten/src/THC/generic/THCTensorRandom.cu @@ -143,10 +143,10 @@ THC_API void THCTensor_(multinomial)(struct THCState *state, // Categories are in the innermost dimension int64_t numDist = - inputSize == 1 ? 1 : THCTensor_(size)(state, prob_dist, 0); + inputSize == 1 ? 1 : THCTensor_(sizeLegacyNoScalars)(state, prob_dist, 0); int64_t numCategoriesLong = - inputSize == 1 ? THCTensor_(size)(state, prob_dist, 0) : - THCTensor_(size)(state, prob_dist, 1); + inputSize == 1 ? THCTensor_(sizeLegacyNoScalars)(state, prob_dist, 0) : + THCTensor_(sizeLegacyNoScalars)(state, prob_dist, 1); // Since the index tensor is float, numCategories cannot exceed max // float integer precision diff --git a/aten/src/THC/generic/THCTensorScatterGather.cu b/aten/src/THC/generic/THCTensorScatterGather.cu index 279005b6650d3..dd7d85f81dbc7 100644 --- a/aten/src/THC/generic/THCTensorScatterGather.cu +++ b/aten/src/THC/generic/THCTensorScatterGather.cu @@ -14,10 +14,8 @@ void THCTensor_(gather)(THCState* state, THCTensor *tensor, THArgCheck(THCudaLongTensor_nDimensionLegacyNoScalars(state, index) == THCTensor_(nDimensionLegacyNoScalars)(state, src), 4, "Index tensor must have same dimensions as input tensor"); - THLongStorage *indexSize = THCudaLongTensor_newSizeOf(state, index); - THArgCheck(THCTensor_(isSize)(state, tensor, indexSize), 4, + THArgCheck(tensor->sizes().equals(index->sizes()), 4, "Index tensor must have the same size as output tensor."); - THLongStorage_free(indexSize); THArgCheck(dim >= 0 && dim < THCTensor_(nDimensionLegacyNoScalars)(state, tensor), 3, "Index dimension is out of bounds"); THArgCheck(THCTensor_(nDimensionLegacyNoScalars)(state, src) == THCTensor_(nDimensionLegacyNoScalars)(state, tensor), 2, @@ -25,7 +23,7 @@ void THCTensor_(gather)(THCState* state, THCTensor *tensor, for (int d = 0; d < THCTensor_(nDimensionLegacyNoScalars)(state, tensor); d++) { if (d != dim) { - THArgCheck(THCTensor_(size)(state, tensor, d) == THCTensor_(size)(state, src, d), 2, + THArgCheck(THCTensor_(sizeLegacyNoScalars)(state, tensor, d) == THCTensor_(sizeLegacyNoScalars)(state, src, d), 2, "Input tensor must have same size as output tensor apart from the specified dimension"); } } @@ -117,13 +115,13 @@ void THCTensor_(scatter)(THCState* state, THCTensor *tensor, int dim, THCudaLong "Input tensor must have same dimensions as output tensor"); for (int d = 0; d < THCTensor_(nDimensionLegacyNoScalars)(state, tensor); d++) { - int64_t indexSizeD = THCudaLongTensor_size(state, index, d); + int64_t indexSizeD = THCudaLongTensor_sizeLegacyNoScalars(state, index, d); if (d != dim) { - THArgCheck(indexSizeD <= THCTensor_(size)(state, tensor, d), 3, + THArgCheck(indexSizeD <= THCTensor_(sizeLegacyNoScalars)(state, tensor, d), 3, "Index tensor must not have larger size than output tensor apart from the specified dimension %d, but got index %s output %s", dim, THCudaLongTensor_sizeDesc(state, index).str, THCTensor_(sizeDesc)(state, tensor).str); } - THArgCheck(indexSizeD <= THCTensor_(size)(state, src, d), 3, + THArgCheck(indexSizeD <= THCTensor_(sizeLegacyNoScalars)(state, src, d), 3, "Index tensor must not have larger size than input tensor, but got index %s input %s", THCudaLongTensor_sizeDesc(state, index).str, THCTensor_(sizeDesc)(state, src).str); } @@ -209,13 +207,13 @@ void THCTensor_(scatterAdd)(THCState* state, THCTensor *tensor, int dim, THCudaL "Input tensor must have same dimensions as output tensor"); for (int d = 0; d < THCTensor_(nDimensionLegacyNoScalars)(state, tensor); d++) { - int64_t indexSizeD = THCudaLongTensor_size(state, index, d); + int64_t indexSizeD = THCudaLongTensor_sizeLegacyNoScalars(state, index, d); if (d != dim) { - THArgCheck(indexSizeD <= THCTensor_(size)(state, tensor, d), 3, + THArgCheck(indexSizeD <= THCTensor_(sizeLegacyNoScalars)(state, tensor, d), 3, "Index tensor must not have larger size than output tensor apart from the specified dimension %d, but got index %s output %s", dim, THCudaLongTensor_sizeDesc(state, index).str, THCTensor_(sizeDesc)(state, tensor).str); } - THArgCheck(indexSizeD <= THCTensor_(size)(state, src, d), 3, + THArgCheck(indexSizeD <= THCTensor_(sizeLegacyNoScalars)(state, src, d), 3, "Index tensor must not have larger size than input tensor, but got index %s input %s", THCudaLongTensor_sizeDesc(state, index).str, THCTensor_(sizeDesc)(state, src).str); } @@ -303,8 +301,8 @@ THCTensor_(scatterFill)(THCState* state, THCTensor *tensor, for (int d = 0; d < THCTensor_(nDimensionLegacyNoScalars)(state, tensor); d++) { if (d != dim) { - THArgCheck(THCTensor_(size)(state, tensor, d) == - THCudaLongTensor_size(state, index, d), 4, + THArgCheck(THCTensor_(sizeLegacyNoScalars)(state, tensor, d) == + THCudaLongTensor_sizeLegacyNoScalars(state, index, d), 4, "Index tensor must have same size as output tensor apart from the specified dimension"); } } diff --git a/aten/src/THC/generic/THCTensorSort.cu b/aten/src/THC/generic/THCTensorSort.cu index 2943a5cf441d7..af81898357b8f 100644 --- a/aten/src/THC/generic/THCTensorSort.cu +++ b/aten/src/THC/generic/THCTensorSort.cu @@ -9,10 +9,8 @@ THC_API void THCTensor_(sortKeyValueInplace)(THCState* state, THCTensor* key, THCudaLongTensor* value, int dim, bool dir) { - THLongStorage *valueSize = THCudaLongTensor_newSizeOf(state, value); - THArgCheck(THCTensor_(isSize)(state, key, valueSize), 2, + THArgCheck(key->sizes().equals(value->sizes()), 2, "Key tensor must have same size as value tensor"); - THLongStorage_free(valueSize); int dims = THCudaLongTensor_nDimensionLegacyNoScalars(state, value); THArgCheck(dims <= MAX_CUTORCH_DIMS, 3, CUTORCH_DIM_WARNING); dims = THCTensor_(nDimensionLegacyNoScalars)(state, key); @@ -24,7 +22,7 @@ THC_API void THCTensor_(sortKeyValueInplace)(THCState* state, return; } - int64_t keySliceSize = THCTensor_(size)(state, key, dim); + int64_t keySliceSize = THCTensor_(sizeLegacyNoScalars)(state, key, dim); ptrdiff_t keySlices = inElements / keySliceSize; // The amount of shared memory and block size is based on @@ -161,8 +159,8 @@ void THCTensor_(sortViaThrust)(THCState* state, int nDims = THCTensor_(nDimensionLegacyAll)(state, input); ptrdiff_t totalElements = THCTensor_(nElement)(state, input); - int64_t sliceSize = THCTensor_(size)(state, input, dim); - int64_t sliceStride = THCTensor_(stride)(state, input, dim); + int64_t sliceSize = THCTensor_(sizeLegacyNoScalars)(state, input, dim); + int64_t sliceStride = THTensor_strideLegacyNoScalars(input, dim); // We perform a vectorized segmented sort in Thrust. // Say we are sorting a (2, 3) tensor. We have in flattened form: @@ -297,7 +295,7 @@ THC_API void THCTensor_(sort)(THCState* state, THLongStorage_free(inputSize); // How large are the slices that we are sorting? - int64_t sliceSize = THCTensor_(size)(state, input, dim); + int64_t sliceSize = THCTensor_(sizeLegacyNoScalars)(state, input, dim); // Workaround: // CUDA 8 uses more shared memory than 7.5 for bitonicSortKVInPlace, diff --git a/aten/src/THC/generic/THCTensorTopK.cu b/aten/src/THC/generic/THCTensorTopK.cu index 24f9a80849f22..c3b3c55b6ce2e 100644 --- a/aten/src/THC/generic/THCTensorTopK.cu +++ b/aten/src/THC/generic/THCTensorTopK.cu @@ -17,7 +17,7 @@ THC_API void THCTensor_(topk)(THCState* state, THArgCheck(dim >= 0 && dim < numDims, 6, "dim not in range"); - int64_t sliceSize = THCTensor_(size)(state, input_, dim); + int64_t sliceSize = THCTensor_(sizeLegacyNoScalars)(state, input_, dim); THArgCheck(k >= 0 && k <= sliceSize, 5, "k not in range for dimension"); THCTensor *input = THCTensor_(newContiguous)(state, input_); diff --git a/aten/src/THCUNN/common.h b/aten/src/THCUNN/common.h index e2a99640ba69b..e8a98079b85b0 100644 --- a/aten/src/THCUNN/common.h +++ b/aten/src/THCUNN/common.h @@ -18,12 +18,10 @@ inline int GET_BLOCKS(const int N) } #define THCUNN_resizeAs_indices(STATE, I1, I2) \ - THLongStorage *size2 = THCTensor_(newSizeOf)(STATE, I2); \ - if (!THCIndexTensor_(isSize)(STATE, I1, size2)) \ + if (!I1->sizes().equals(I2->sizes())) \ { \ - THCudaLongTensor_resize(STATE, I1, size2, NULL); \ - } \ - THLongStorage_free(size2); + THCudaLongTensor_resizeAs(STATE, I1, I2); \ + } #define THCUNN_check_shape(STATE, I1, I2) \ if (I1 != NULL && I2 != NULL && !THCTensor_(isSameSizeAs)(STATE, I1, I2)) \ @@ -36,15 +34,13 @@ inline int GET_BLOCKS(const int N) #define THCUNN_check_shape_indices(STATE, I1, I2) \ - THLongStorage *size2 = THCTensor_(newSizeOf)(STATE, I2); \ - if (!THCIndexTensor_(isSize)(STATE, I1, size2)) \ + if (!I1->sizes().equals(I2->sizes())) \ { \ THCDescBuff s1 = THCIndexTensor_(sizeDesc)(STATE, I1); \ THCDescBuff s2 = THCTensor_(sizeDesc)(STATE, I2); \ THError(#I1 " and " #I2 " shapes do not match: " \ #I1 " %s, " #I2 " %s", s1.str, s2.str); \ - } \ - THLongStorage_free(size2); + } #define THCUNN_check_nElement(STATE, I1, I2) \ if (I1 != NULL && I2 != NULL ) { \ diff --git a/aten/src/THNN/THNN.h b/aten/src/THNN/THNN.h index e216e6204196b..bf0f156579b78 100644 --- a/aten/src/THNN/THNN.h +++ b/aten/src/THNN/THNN.h @@ -19,14 +19,6 @@ typedef int64_t THIndex_t; typedef int32_t THInteger_t; typedef void THNNState; -#define THNN_resizeAs_indices(I1, I2) \ - THLongStorage *size2 = THIndexTensor_(newSizeOf)(I2); \ - if (!THTensor_(isSize)(I1, size2)) \ - { \ - THTensor_(resize)(I1, size2, NULL); \ - } \ - THLongStorage_free(size2); - #include "generic/THNN.h" #include diff --git a/aten/src/THNN/generic/AbsCriterion.c b/aten/src/THNN/generic/AbsCriterion.c index 73552a2bc3b50..05f14773ef565 100644 --- a/aten/src/THNN/generic/AbsCriterion.c +++ b/aten/src/THNN/generic/AbsCriterion.c @@ -54,7 +54,7 @@ void THNN_(AbsCriterion_updateGradInput)( } THNN_CHECK_DIM_SIZE(gradOutput, 1, 0, 1); - real norm = (reduction == Reduction::ElementwiseMean ? 1./((real)THTensor_(nElement)(input)) : 1.) * THTensor_(fastGet1d)(gradOutput, 0); + real norm = (reduction == Reduction::ElementwiseMean ? 1./((real)THTensor_(nElement)(input)) : 1.) * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); TH_TENSOR_APPLY3(real, gradInput, real, input, real, target, *gradInput_data = (*input_data - *target_data) >= 0 ? norm : -norm; diff --git a/aten/src/THNN/generic/BCECriterion.c b/aten/src/THNN/generic/BCECriterion.c index f3f74ca108128..079493e305650 100644 --- a/aten/src/THNN/generic/BCECriterion.c +++ b/aten/src/THNN/generic/BCECriterion.c @@ -106,7 +106,7 @@ void THNN_(BCECriterion_updateGradInput)( TH_TENSOR_APPLY3(real, gradInput, real, input, real, target, real x = *input_data; real y = *target_data; - *gradInput_data = - norm * (y - x) / ((1. - x + EPS) * (x + EPS)) * THTensor_(fastGet1d)(gradOutput, 0); + *gradInput_data = - norm * (y - x) / ((1. - x + EPS) * (x + EPS)) * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); ); if(weights) diff --git a/aten/src/THNN/generic/ClassNLLCriterion.c b/aten/src/THNN/generic/ClassNLLCriterion.c index 7db0531d60d1e..c635a278d519d 100644 --- a/aten/src/THNN/generic/ClassNLLCriterion.c +++ b/aten/src/THNN/generic/ClassNLLCriterion.c @@ -37,14 +37,14 @@ void THNN_(ClassNLLCriterion_updateOutput)( int i; #pragma omp parallel for private(i) for (i = 0; i < batch_size; i++) { - int cur_target = THLongTensor_fastGet1d(target, i) - TH_INDEX_BASE; + int cur_target = THLongTensor_fastGetLegacy1dNoScalars(target, i) - TH_INDEX_BASE; if (cur_target >= 0 && cur_target < n_classes) { if (cur_target == ignore_index) { THTensor_(fastSet1d)(output, i, 0.0f); continue; } - real cur_weight = weights ? THTensor_(fastGet1d)(weights, cur_target) : 1.0f; + real cur_weight = weights ? THTensor_(fastGetLegacy1dNoScalars)(weights, cur_target) : 1.0f; THTensor_(fastSet1d)(output, i, -THTensor_(fastGet2d)(input, i, cur_target) * cur_weight); } else { int tmp = -1; @@ -151,12 +151,12 @@ void THNN_(ClassNLLCriterion_updateGradInput)( int i; #pragma omp parallel for private(i) for (i = 0; i < batch_size; i++) { - int cur_target = THLongTensor_fastGet1d(target, i) - TH_INDEX_BASE; + int cur_target = THLongTensor_fastGetLegacy1dNoScalars(target, i) - TH_INDEX_BASE; if (cur_target == ignore_index) { continue; } - real weight = weights ? THTensor_(fastGet1d)(weights, cur_target) : 1.0f; - THTensor_(fastSet2d)(gradInput, i, cur_target, -weight * THTensor_(fastGet1d)(gradOutput, i)); + real weight = weights ? THTensor_(fastGetLegacy1dNoScalars)(weights, cur_target) : 1.0f; + THTensor_(fastSet2d)(gradInput, i, cur_target, -weight * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, i)); } return; } diff --git a/aten/src/THNN/generic/DistKLDivCriterion.c b/aten/src/THNN/generic/DistKLDivCriterion.c index 823360812cae5..64dfb345fe762 100644 --- a/aten/src/THNN/generic/DistKLDivCriterion.c +++ b/aten/src/THNN/generic/DistKLDivCriterion.c @@ -57,7 +57,7 @@ void THNN_(DistKLDivCriterion_updateGradInput)( real norm = (reduction == Reduction::ElementwiseMean ? 1./((real)THTensor_(nElement)(input)) : 1.); TH_TENSOR_APPLY3(real, gradInput, real, input, real, target, - *gradInput_data = *target_data > 0 ? norm * (-*target_data) * THTensor_(fastGet1d)(gradOutput, 0) : 0; + *gradInput_data = *target_data > 0 ? norm * (-*target_data) * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0) : 0; ); } diff --git a/aten/src/THNN/generic/MultiLabelMarginCriterion.c b/aten/src/THNN/generic/MultiLabelMarginCriterion.c index a18252b06914d..7a28ee1cadb76 100644 --- a/aten/src/THNN/generic/MultiLabelMarginCriterion.c +++ b/aten/src/THNN/generic/MultiLabelMarginCriterion.c @@ -43,7 +43,9 @@ void THNN_(MultiLabelMarginCriterion_updateOutput)( input_data = THTensor_(data)(input); target_data = THIndexTensor_(data)(target); - THNN_resizeAs_indices(isTarget, target); + if (!isTarget->sizes().equals(target->sizes())) { + THTensor_(resizeNd)(isTarget, target->dim(), THTensor_getSizePtr(target), nullptr); + } THTensor_(zero)(isTarget); isTarget_data = THTensor_(data)(isTarget); @@ -232,7 +234,7 @@ void THNN_(MultiLabelMarginCriterion_updateGradInput)( THNN_CHECK_DIM_SIZE(gradOutput, 1, 0, 1); for (t = 0; t < nframe*dim; t++) { - gradInput_data[t] *= THTensor_(fastGet1d)(gradOutput, 0); + gradInput_data[t] *= THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); } } else @@ -242,7 +244,7 @@ void THNN_(MultiLabelMarginCriterion_updateGradInput)( { for (d = 0; d < dim; d++) { - gradInput_data[t * dim + d] *= THTensor_(fastGet1d)(gradOutput, t); + gradInput_data[t * dim + d] *= THTensor_(fastGetLegacy1dNoScalars)(gradOutput, t); } } } diff --git a/aten/src/THNN/generic/MultiMarginCriterion.c b/aten/src/THNN/generic/MultiMarginCriterion.c index 2c8f38be23eb3..92dbc520907ba 100644 --- a/aten/src/THNN/generic/MultiMarginCriterion.c +++ b/aten/src/THNN/generic/MultiMarginCriterion.c @@ -199,7 +199,7 @@ void THNN_(MultiMarginCriterion_updateGradInput)( { THNN_CHECK_DIM_SIZE(gradOutput, 1, 0, 1); for (t = 0; t < nframe * dim; t++) { - gradInput_data[t] *= THTensor_(fastGet1d)(gradOutput, 0); + gradInput_data[t] *= THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); } } else @@ -209,7 +209,7 @@ void THNN_(MultiMarginCriterion_updateGradInput)( { for (d = 0; d < dim; d++) { - gradInput_data[t * dim + d] *= THTensor_(fastGet1d)(gradOutput, t); + gradInput_data[t * dim + d] *= THTensor_(fastGetLegacy1dNoScalars)(gradOutput, t); } } } diff --git a/aten/src/THNN/generic/SmoothL1Criterion.c b/aten/src/THNN/generic/SmoothL1Criterion.c index b9eca659eeeb2..e8b2398483c20 100644 --- a/aten/src/THNN/generic/SmoothL1Criterion.c +++ b/aten/src/THNN/generic/SmoothL1Criterion.c @@ -64,7 +64,7 @@ void THNN_(SmoothL1Criterion_updateGradInput)( } THNN_CHECK_DIM_SIZE(gradOutput, 1, 0, 1); - real norm = (reduction == Reduction::ElementwiseMean ? 1./((real)THTensor_(nElement)(input)) : 1.) * THTensor_(fastGet1d)(gradOutput, 0); + real norm = (reduction == Reduction::ElementwiseMean ? 1./((real)THTensor_(nElement)(input)) : 1.) * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); TH_TENSOR_APPLY3(real, gradInput, real, input, real, target, real x = *input_data - *target_data; diff --git a/aten/src/THNN/generic/SoftMarginCriterion.c b/aten/src/THNN/generic/SoftMarginCriterion.c index 8fb31f9952e32..08c879e169e11 100644 --- a/aten/src/THNN/generic/SoftMarginCriterion.c +++ b/aten/src/THNN/generic/SoftMarginCriterion.c @@ -59,7 +59,7 @@ void THNN_(SoftMarginCriterion_updateGradInput)( TH_TENSOR_APPLY3(real, gradInput, real, input, real, target, real z = exp(-*target_data * *input_data); - *gradInput_data = -norm*(*target_data)*z/(1. + z) * THTensor_(fastGet1d)(gradOutput, 0);) + *gradInput_data = -norm*(*target_data)*z/(1. + z) * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0);) } #endif diff --git a/aten/src/THNN/generic/SpatialClassNLLCriterion.c b/aten/src/THNN/generic/SpatialClassNLLCriterion.c index 967f79c370924..246ba08998703 100644 --- a/aten/src/THNN/generic/SpatialClassNLLCriterion.c +++ b/aten/src/THNN/generic/SpatialClassNLLCriterion.c @@ -78,7 +78,7 @@ void THNN_(SpatialClassNLLCriterion_updateOutput)( continue; } real value = THTensor_(fastGet4d)(input, b, cur_target, h, w); - real weight = weights ? THTensor_(fastGet1d)(weights, cur_target) : 1.0f; + real weight = weights ? THTensor_(fastGetLegacy1dNoScalars)(weights, cur_target) : 1.0f; THTensor_(fastSet3d)(output, b, h, w, -value * weight); } } @@ -161,7 +161,7 @@ void THNN_(SpatialClassNLLCriterion_updateGradInput)( if (cur_target == ignore_index) { continue; } - real value = -(weights ? THTensor_(fastGet1d)(weights, cur_target) : 1.0f); + real value = -(weights ? THTensor_(fastGetLegacy1dNoScalars)(weights, cur_target) : 1.0f); real gradOutput_value = THTensor_(fastGet3d)(gradOutput, b, h, w); THTensor_(fastSet4d)(gradInput, b, cur_target, h, w, value * gradOutput_value); } @@ -201,7 +201,7 @@ void THNN_(SpatialClassNLLCriterion_updateGradInput)( int index = b * sample_size + cur_target * map_size + elem; gradInput_data[index] = - -(weights ? weights_data[cur_target] : 1.0f) / normalize * THTensor_(fastGet1d)(gradOutput, 0); + -(weights ? weights_data[cur_target] : 1.0f) / normalize * THTensor_(fastGetLegacy1dNoScalars)(gradOutput, 0); } } diff --git a/aten/src/THNN/init.cpp b/aten/src/THNN/init.cpp index c77cd76d54ec8..e886272298a48 100644 --- a/aten/src/THNN/init.cpp +++ b/aten/src/THNN/init.cpp @@ -17,16 +17,12 @@ } #define THNN_CHECK_SHAPE_INDICES(I1, I2) \ - THLongStorage *size2 = THLongTensor_newSizeOf(I2); \ - if (I1 != NULL && I2 != NULL && !THTensor_(isSize)(I1, size2)) \ + if (I1 != NULL && I2 != NULL && !I1->sizes().equals(I2->sizes())) \ { \ THDescBuff s1 = THTensor_(sizeDesc)(I1); \ THDescBuff s2 = THLongTensor_sizeDesc(I2); \ - THLongStorage_free(size2); \ THError(#I1 " and " #I2 " shapes do not match: " \ #I1 " %s, " #I2 " %s", s1.str, s2.str); \ - } else { \ - THLongStorage_free(size2); \ } #define THNN_CHECK_NELEMENT(I1, I2) \ diff --git a/caffe2/core/blob_serialization.cc b/caffe2/core/blob_serialization.cc index b870aa39067e6..df5ffbfce98f1 100644 --- a/caffe2/core/blob_serialization.cc +++ b/caffe2/core/blob_serialization.cc @@ -322,7 +322,7 @@ void TensorSerializer::StoreDeviceDetail( // The actual serialization registry objects. CAFFE_DEFINE_TYPED_REGISTRY( BlobSerializerRegistry, - CaffeTypeId, + TypeIdentifier, BlobSerializerBase, std::unique_ptr); diff --git a/caffe2/core/blob_serialization.h b/caffe2/core/blob_serialization.h index 18cb95d541b4b..8f2378886db42 100644 --- a/caffe2/core/blob_serialization.h +++ b/caffe2/core/blob_serialization.h @@ -26,13 +26,13 @@ constexpr auto kChunkIdSeparator = "#%"; // The Blob serialization registry and serializer creator functions. CAFFE_DECLARE_TYPED_REGISTRY( BlobSerializerRegistry, - CaffeTypeId, + TypeIdentifier, BlobSerializerBase, std::unique_ptr); #define REGISTER_BLOB_SERIALIZER(id, ...) \ CAFFE_REGISTER_TYPED_CLASS(BlobSerializerRegistry, id, __VA_ARGS__) // Creates an operator with the given operator definition. -inline unique_ptr CreateSerializer(CaffeTypeId id) { +inline unique_ptr CreateSerializer(TypeIdentifier id) { return BlobSerializerRegistry()->Create(id); } diff --git a/caffe2/core/blob_stats.cc b/caffe2/core/blob_stats.cc index 057e966aa319f..0c9d0e1bea26e 100644 --- a/caffe2/core/blob_stats.cc +++ b/caffe2/core/blob_stats.cc @@ -2,7 +2,7 @@ namespace caffe2 { -const BlobStatGetter* BlobStatRegistry::get(CaffeTypeId id) { +const BlobStatGetter* BlobStatRegistry::get(TypeIdentifier id) { auto it = map_.find(id); if (it == map_.end()) { return nullptr; @@ -16,7 +16,7 @@ BlobStatRegistry& BlobStatRegistry::instance() { } void BlobStatRegistry::doRegister( - CaffeTypeId id, + TypeIdentifier id, std::unique_ptr&& v) { // don't use CAFFE_ENFORCE_EQ to avoid static initialization order fiasco. if (map_.count(id) > 0) { diff --git a/caffe2/core/blob_stats.h b/caffe2/core/blob_stats.h index f4ef21d0cfb75..178bda0edd78b 100644 --- a/caffe2/core/blob_stats.h +++ b/caffe2/core/blob_stats.h @@ -15,8 +15,8 @@ struct BlobStatGetter { struct BlobStatRegistry { private: - std::unordered_map> map_; - void doRegister(CaffeTypeId id, std::unique_ptr&& v); + std::unordered_map> map_; + void doRegister(TypeIdentifier id, std::unique_ptr&& v); public: template @@ -27,7 +27,7 @@ struct BlobStatRegistry { } }; - const BlobStatGetter* get(CaffeTypeId id); + const BlobStatGetter* get(TypeIdentifier id); static BlobStatRegistry& instance(); }; diff --git a/caffe2/core/dispatch/DispatchKey.h b/caffe2/core/dispatch/DispatchKey.h index 6622f553b64f0..fe309b7f709b7 100644 --- a/caffe2/core/dispatch/DispatchKey.h +++ b/caffe2/core/dispatch/DispatchKey.h @@ -16,8 +16,8 @@ struct TensorParameterDispatchKey final { // note: This dispatch key structure is not final yet and will change. Don't rely on it. DeviceTypeId deviceTypeId; LayoutId layoutId; - // TODO Move this CaffeTypeId to c10 namespace - caffe2::CaffeTypeId dataType; + // TODO Move this TypeIdentifier to c10 namespace + caffe2::TypeIdentifier dataType; }; inline constexpr bool operator==(const TensorParameterDispatchKey& lhs, const TensorParameterDispatchKey& rhs) { return lhs.deviceTypeId == rhs.deviceTypeId && lhs.layoutId == rhs.layoutId && lhs.dataType == rhs.dataType; @@ -34,7 +34,7 @@ namespace std { struct hash { // TODO constexpr hashing size_t operator()(const c10::details::TensorParameterDispatchKey& obj) const { - return std::hash()(obj.deviceTypeId) ^ std::hash()(obj.layoutId) ^ std::hash()(obj.dataType); + return std::hash()(obj.deviceTypeId) ^ std::hash()(obj.layoutId) ^ std::hash()(obj.dataType); } }; } // namespace std diff --git a/caffe2/core/nomnigraph/include/nomnigraph/Graph/Graph.h b/caffe2/core/nomnigraph/include/nomnigraph/Graph/Graph.h index aab127d8c56e1..cb7b90059a156 100644 --- a/caffe2/core/nomnigraph/include/nomnigraph/Graph/Graph.h +++ b/caffe2/core/nomnigraph/include/nomnigraph/Graph/Graph.h @@ -189,7 +189,7 @@ class Subgraph { return nodes_; } - const size_t getNodesCount() const { + size_t getNodesCount() const { return (size_t)nodes_.size(); } diff --git a/caffe2/core/plan_executor.cc b/caffe2/core/plan_executor.cc index 1944874437d73..7e8490d700a67 100644 --- a/caffe2/core/plan_executor.cc +++ b/caffe2/core/plan_executor.cc @@ -100,7 +100,7 @@ std::function getContinuationTest( // if the blob doesn't exist or is not initiaized, return false inline bool getShouldStop(const Blob* b) { - if (!b || b->meta().id() == CaffeTypeId::uninitialized()) { // not exist or uninitialized + if (!b || b->meta().id() == TypeIdentifier::uninitialized()) { // not exist or uninitialized return false; } diff --git a/caffe2/core/tensor.cc b/caffe2/core/tensor.cc index bd6e6687087f2..ed00a4578d652 100644 --- a/caffe2/core/tensor.cc +++ b/caffe2/core/tensor.cc @@ -68,10 +68,10 @@ TypeMeta GetTensorType(const void* c) { } // TODO(jerryzh): Remove -static CaffeMap type_call_registry_{ +static CaffeMap type_call_registry_{ {TypeMeta::Id(), GetTensorType}}; -TypeCall GetTypeCallFunction(CaffeTypeId id) { +TypeCall GetTypeCallFunction(TypeIdentifier id) { auto f = type_call_registry_.find(id); if (f == type_call_registry_.end()) { return nullptr; @@ -79,7 +79,7 @@ TypeCall GetTypeCallFunction(CaffeTypeId id) { return f->second; } -void RegisterTypeCallFunction(CaffeTypeId id, TypeCall c) { +void RegisterTypeCallFunction(TypeIdentifier id, TypeCall c) { type_call_registry_[id] = c; } @@ -98,12 +98,12 @@ vector GetTensorInfo( } // since we only have one tensor, probably need to remove this at some point? -static CaffeMap tensor_info_call_registry_{ +static CaffeMap tensor_info_call_registry_{ {TypeMeta::Id(), GetTensorInfo}}; // TODO: Remove this code in a separate diff, since we only have one // GetTensorInfo function now -TensorInfoCall GetTensorInfoFunction(CaffeTypeId id) { +TensorInfoCall GetTensorInfoFunction(TypeIdentifier id) { auto f = tensor_info_call_registry_.find(id); if (f == tensor_info_call_registry_.end()) { return nullptr; @@ -111,7 +111,7 @@ TensorInfoCall GetTensorInfoFunction(CaffeTypeId id) { return f->second; } -void RegisterTensorInfoFunction(CaffeTypeId id, TensorInfoCall c) { +void RegisterTensorInfoFunction(TypeIdentifier id, TensorInfoCall c) { tensor_info_call_registry_[id] = c; } diff --git a/caffe2/core/tensor.h b/caffe2/core/tensor.h index 757876c1f51c6..2cf4ed8fdce03 100644 --- a/caffe2/core/tensor.h +++ b/caffe2/core/tensor.h @@ -473,7 +473,7 @@ class Tensor { Deleter d = nullptr) { meta_ = meta; CAFFE_ENFORCE_WITH_CALLER( - meta_.id() != CaffeTypeId::uninitialized(), + meta_.id() != TypeIdentifier::uninitialized(), "To share with a raw external pointer you need to have meta " "already set."); CAFFE_ENFORCE_WITH_CALLER( @@ -600,7 +600,7 @@ class Tensor { */ inline void* raw_mutable_data() { CAFFE_ENFORCE_WITH_CALLER( - meta_.id() != CaffeTypeId::uninitialized(), + meta_.id() != TypeIdentifier::uninitialized(), "Calling raw_mutable_data() without meta, but the current meta is " "of unknown type."); return raw_mutable_data(meta_); @@ -829,8 +829,8 @@ constexpr int k_limit_default_ = 1000; // Type call registry typedef TypeMeta (*TypeCall)(const void*); -TypeCall GetTypeCallFunction(CaffeTypeId id); -void RegisterTypeCallFunction(CaffeTypeId id, TypeCall c); +TypeCall GetTypeCallFunction(TypeIdentifier id); +void RegisterTypeCallFunction(TypeIdentifier id, TypeCall c); // Shape call registry typedef vector (*TensorInfoCall)( @@ -838,8 +838,8 @@ typedef vector (*TensorInfoCall)( bool* shares_data, size_t* capacity, DeviceOption* device); -TensorInfoCall GetTensorInfoFunction(CaffeTypeId id); -void RegisterTensorInfoFunction(CaffeTypeId id, TensorInfoCall c); +TensorInfoCall GetTensorInfoFunction(TypeIdentifier id); +void RegisterTensorInfoFunction(TypeIdentifier id, TensorInfoCall c); // resize helper function void TensorVectorResize( diff --git a/caffe2/core/typeid.cc b/caffe2/core/typeid.cc index ba81e3babc6d0..d4c5294f4b5d3 100644 --- a/caffe2/core/typeid.cc +++ b/caffe2/core/typeid.cc @@ -13,8 +13,8 @@ using std::string; namespace caffe2 { -std::unordered_map& gTypeNames() { - static std::unordered_map g_type_names; +std::unordered_map& gTypeNames() { + static std::unordered_map g_type_names; return g_type_names; } @@ -59,14 +59,14 @@ void TypeMeta::_ThrowRuntimeTypeLogicError(const std::string& msg) { CAFFE_THROW(msg); } -CaffeTypeId CaffeTypeId::createTypeId() { - static std::atomic counter( +TypeIdentifier TypeIdentifier::createTypeId() { + static std::atomic counter( TypeMeta::Id<_CaffeHighestPreallocatedTypeId>().underlyingId()); - const CaffeTypeId::underlying_type new_value = ++counter; - if (new_value == std::numeric_limits::max()) { - throw std::logic_error("Ran out of available type ids. If you need more than 2^16 CAFFE_KNOWN_TYPEs, we need to increase CaffeTypeId to use more than 16 bit."); + const TypeIdentifier::underlying_type new_value = ++counter; + if (new_value == std::numeric_limits::max()) { + throw std::logic_error("Ran out of available type ids. If you need more than 2^16 CAFFE_KNOWN_TYPEs, we need to increase TypeIdentifier to use more than 16 bit."); } - return CaffeTypeId(new_value); + return TypeIdentifier(new_value); } CAFFE_DEFINE_KNOWN_TYPE(Tensor); @@ -103,7 +103,7 @@ namespace { // intended to be only instantiated once here. struct UninitializedTypeNameRegisterer { UninitializedTypeNameRegisterer() { - gTypeNames()[CaffeTypeId::uninitialized()] = "nullptr (uninitialized)"; + gTypeNames()[TypeIdentifier::uninitialized()] = "nullptr (uninitialized)"; } }; static UninitializedTypeNameRegisterer g_uninitialized_type_name_registerer; diff --git a/caffe2/core/typeid.h b/caffe2/core/typeid.h index facea9fa64d2f..609c67a61dbf2 100644 --- a/caffe2/core/typeid.h +++ b/caffe2/core/typeid.h @@ -19,10 +19,10 @@ #include "ATen/core/IdWrapper.h" namespace caffe2 { -class CaffeTypeId; +class TypeIdentifier; } -std::ostream& operator<<(std::ostream& stream, caffe2::CaffeTypeId typeId); +std::ostream& operator<<(std::ostream& stream, caffe2::TypeIdentifier typeId); namespace caffe2 { @@ -30,43 +30,43 @@ class TypeMeta; /** * A type id is a unique id for a given C++ type. - * You need to register your types using CAFFE_KNOWN_TYPE(MyType) to be able to use CaffeTypeId with custom types. + * You need to register your types using CAFFE_KNOWN_TYPE(MyType) to be able to use TypeIdentifier with custom types. * This is for example used to store the dtype of tensors. */ -class CaffeTypeId final : public at::IdWrapper { +class TypeIdentifier final : public at::IdWrapper { public: - static CaffeTypeId createTypeId(); + static TypeIdentifier createTypeId(); - friend std::ostream& ::operator<<(std::ostream& stream, CaffeTypeId typeId); - friend bool operator<(CaffeTypeId lhs, CaffeTypeId rhs); + friend std::ostream& ::operator<<(std::ostream& stream, TypeIdentifier typeId); + friend bool operator<(TypeIdentifier lhs, TypeIdentifier rhs); // This is 8, because 0 is uint8_t (due to ScalarType BC constraint) - static constexpr CaffeTypeId uninitialized() { - return CaffeTypeId(8); + static constexpr TypeIdentifier uninitialized() { + return TypeIdentifier(8); } private: - constexpr explicit CaffeTypeId(uint16_t id): IdWrapper(id) {} + constexpr explicit TypeIdentifier(uint16_t id): IdWrapper(id) {} friend class TypeMeta; }; // Allow usage in std::map / std::set // TODO Disallow this and rather use std::unordered_map/set everywhere -inline bool operator<(CaffeTypeId lhs, CaffeTypeId rhs) { +inline bool operator<(TypeIdentifier lhs, TypeIdentifier rhs) { return lhs.underlyingId() < rhs.underlyingId(); } } -AT_DEFINE_HASH_FOR_IDWRAPPER(caffe2::CaffeTypeId) +AT_DEFINE_HASH_FOR_IDWRAPPER(caffe2::TypeIdentifier) -inline std::ostream& operator<<(std::ostream& stream, caffe2::CaffeTypeId typeId) { +inline std::ostream& operator<<(std::ostream& stream, caffe2::TypeIdentifier typeId) { return stream << typeId.underlyingId(); } namespace caffe2 { -std::unordered_map& gTypeNames(); +std::unordered_map& gTypeNames(); std::unordered_set& gRegisteredTypeNames(); // A utility function to demangle a function name. @@ -95,7 +95,7 @@ std::mutex& gTypeRegistrationMutex(); template struct TypeNameRegisterer { - TypeNameRegisterer(CaffeTypeId id, const std::string& literal_name) { + TypeNameRegisterer(TypeIdentifier id, const std::string& literal_name) { std::lock_guard guard(gTypeRegistrationMutex()); #ifdef __GXX_RTTI (void)literal_name; @@ -141,7 +141,7 @@ class TypeMeta { * type, use TypeMeta::Make(). */ TypeMeta() noexcept - : id_(CaffeTypeId::uninitialized()), itemsize_(0), ctor_(nullptr), copy_(nullptr), dtor_(nullptr) {} + : id_(TypeIdentifier::uninitialized()), itemsize_(0), ctor_(nullptr), copy_(nullptr), dtor_(nullptr) {} /** * Copy constructor. @@ -159,7 +159,7 @@ class TypeMeta { // TypeMeta can only be created by Make, making sure that we do not // create incorrectly mixed up TypeMeta objects. TypeMeta( - CaffeTypeId i, + TypeIdentifier i, size_t s, PlacementNew* ctor, TypedCopy* copy, @@ -176,7 +176,7 @@ class TypeMeta { /** * Returns the type id. */ - const CaffeTypeId& id() const noexcept { + const TypeIdentifier& id() const noexcept { return id_; } /** @@ -229,7 +229,7 @@ class TypeMeta { * is generated during run-time. Do NOT serialize the id for storage. */ template - CAFFE2_API static CaffeTypeId Id(); + CAFFE2_API static TypeIdentifier Id(); /** * Returns the item size of the type. This is equivalent to sizeof(T). @@ -356,7 +356,7 @@ class TypeMeta { } private: - CaffeTypeId id_; + TypeIdentifier id_; size_t itemsize_; PlacementNew* ctor_; TypedCopy* copy_; @@ -393,16 +393,16 @@ inline bool operator!=(const TypeMeta& lhs, const TypeMeta& rhs) noexcept { #ifdef _MSC_VER #define CAFFE_KNOWN_TYPE(T) \ template <> \ - CAFFE2_EXPORT CaffeTypeId TypeMeta::Id() { \ - static const CaffeTypeId type_id = CaffeTypeId::createTypeId(); \ + CAFFE2_EXPORT TypeIdentifier TypeMeta::Id() { \ + static const TypeIdentifier type_id = TypeIdentifier::createTypeId(); \ static TypeNameRegisterer registerer(type_id, #T); \ return type_id; \ } #else // _MSC_VER #define CAFFE_KNOWN_TYPE(T) \ template <> \ - CaffeTypeId TypeMeta::Id() { \ - static const CaffeTypeId type_id = CaffeTypeId::createTypeId(); \ + TypeIdentifier TypeMeta::Id() { \ + static const TypeIdentifier type_id = TypeIdentifier::createTypeId(); \ static TypeNameRegisterer registerer(type_id, #T); \ return type_id; \ } @@ -417,14 +417,14 @@ inline bool operator!=(const TypeMeta& lhs, const TypeMeta& rhs) noexcept { #ifdef _MSC_VER #define CAFFE_DECLARE_KNOWN_TYPE(PreallocatedId, T) \ template <> \ - inline CAFFE2_EXPORT CaffeTypeId TypeMeta::Id() { \ - return CaffeTypeId(PreallocatedId); \ + inline CAFFE2_EXPORT TypeIdentifier TypeMeta::Id() { \ + return TypeIdentifier(PreallocatedId); \ } #else // _MSC_VER #define CAFFE_DECLARE_KNOWN_TYPE(PreallocatedId, T) \ template <> \ - inline CaffeTypeId TypeMeta::Id() { \ - return CaffeTypeId(PreallocatedId); \ + inline TypeIdentifier TypeMeta::Id() { \ + return TypeIdentifier(PreallocatedId); \ } #endif diff --git a/caffe2/core/types.cc b/caffe2/core/types.cc index c7f3b6a31754d..e1c907a19a55a 100644 --- a/caffe2/core/types.cc +++ b/caffe2/core/types.cc @@ -13,7 +13,7 @@ CAFFE_KNOWN_TYPE(caffe2::float16); TensorProto::DataType TypeMetaToDataType(const TypeMeta& meta) { static_assert(sizeof(int) == 4, "int in this compiler does not equal to 4 bytes."); - static std::map data_type_map { + static std::map data_type_map { {TypeMeta::Id(), TensorProto_DataType_FLOAT}, {TypeMeta::Id(), TensorProto_DataType_INT32}, // BYTE does not have a type meta to proto mapping: we should diff --git a/caffe2/experiments/operators/tt_pad_op.h b/caffe2/experiments/operators/tt_pad_op.h index c85101542ce4f..6d4ab57a9d66a 100644 --- a/caffe2/experiments/operators/tt_pad_op.h +++ b/caffe2/experiments/operators/tt_pad_op.h @@ -52,7 +52,7 @@ class TTPadOp final : public Operator { TIndex padded_dim0 = (X_dim0 / scale_ + 1) * scale_; auto dim0_diff = padded_dim0 - X_dim0; // set growthPct to the upper bound percentage: (100 * scale_ / X_dim0) - X_pad->template Extend(dim0_diff, 100 * scale_ / X_dim0, &context_); + X_pad->Extend(dim0_diff, 100 * scale_ / X_dim0, &context_); auto* X_pad_data = X_pad->template mutable_data(); TIndex X_size = X_dim0 * X_dim1; diff --git a/caffe2/operators/quant_decode_op.h b/caffe2/operators/quant_decode_op.h index 8068b2e00510e..219d579f81a00 100644 --- a/caffe2/operators/quant_decode_op.h +++ b/caffe2/operators/quant_decode_op.h @@ -73,7 +73,7 @@ inline void DecodeGeneral( Tensor* outDecoded, bool resizeOnly) { const static std::map< - std::pair, + std::pair, std::function= 0, ( 'Lars offset must be nonnegative, got {}'.format(self.lars)) + wd, trust, lr_max = self.create_lars_inputs( + param_init_net, 0.0, 1.0, np.finfo(np.float32).max) lr_lars_multiplier = net.Lars( - [param, grad], + [param, grad, wd, trust, lr_max], self.make_unique_blob_name(str(param) + "_lars"), - offset=self.lars) + offset=self.lars, + lr_min=0.0) current_scope = scope.CurrentDeviceScope() self._add_local_lr_multiplier( lr_lars_multiplier, @@ -520,10 +531,14 @@ def _run(self, net, param_init_net, param_info): if self.lars is not None and not isinstance(grad, core.GradientSlice): assert self.lars >= 0, ( 'Lars offset must be nonnegative, got {}'.format(self.lars)) + wd, trust, lr_max = self.create_lars_inputs( + param_init_net, 0.0, 1.0, np.finfo(np.float32).max) lr_lars_multiplier = net.Lars( - [param, grad], + [param, grad, wd, trust, lr_max], self.make_unique_blob_name(str(param) + "_lars"), - offset=self.lars) + offset=self.lars, + lr_min=0.0) + current_scope = scope.CurrentDeviceScope() self._add_local_lr_multiplier( lr_lars_multiplier, @@ -641,10 +656,13 @@ def _run(self, net, param_init_net, param_info): if self.lars is not None and not isinstance(grad, core.GradientSlice): assert self.lars >= 0, ( 'Lars offset must be nonnegative, got {}'.format(self.lars)) + wd, trust, lr_max = self.create_lars_inputs( + param_init_net, 0.0, 1.0, np.finfo(np.float32).max) lr_lars_multiplier = net.Lars( - [param, grad], + [param, grad, wd, trust, lr_max], self.make_unique_blob_name(str(param) + "_lars"), - offset=self.lars) + offset=self.lars, + lr_min=0.0) current_scope = scope.CurrentDeviceScope() self._add_local_lr_multiplier( lr_lars_multiplier, diff --git a/caffe2/python/pybind_state.cc b/caffe2/python/pybind_state.cc index 2544ef314050c..8610eb0490377 100644 --- a/caffe2/python/pybind_state.cc +++ b/caffe2/python/pybind_state.cc @@ -53,7 +53,7 @@ BlobFeederBase::~BlobFeederBase() {} CAFFE_DEFINE_TYPED_REGISTRY( BlobFetcherRegistry, - CaffeTypeId, + TypeIdentifier, BlobFetcherBase, std::unique_ptr); CAFFE_DEFINE_TYPED_REGISTRY( @@ -82,7 +82,7 @@ static_assert( "We make an assumption that int is always int32 for numpy " "type mapping."); int CaffeToNumpyType(const TypeMeta& meta) { - static std::map numpy_type_map{ + static std::map numpy_type_map{ {TypeMeta::Id(), NPY_BOOL}, {TypeMeta::Id(), NPY_DOUBLE}, {TypeMeta::Id(), NPY_FLOAT}, diff --git a/caffe2/python/pybind_state.h b/caffe2/python/pybind_state.h index 894c420afa94b..ed15bb17685a1 100644 --- a/caffe2/python/pybind_state.h +++ b/caffe2/python/pybind_state.h @@ -62,12 +62,12 @@ class BlobFeederBase { CAFFE2_EXPORT CAFFE_DECLARE_TYPED_REGISTRY( BlobFetcherRegistry, - CaffeTypeId, + TypeIdentifier, BlobFetcherBase, std::unique_ptr); #define REGISTER_BLOB_FETCHER(id, ...) \ CAFFE_REGISTER_TYPED_CLASS(BlobFetcherRegistry, id, __VA_ARGS__) -inline unique_ptr CreateFetcher(CaffeTypeId id) { +inline unique_ptr CreateFetcher(TypeIdentifier id) { return BlobFetcherRegistry()->Create(id); } @@ -168,7 +168,7 @@ class TensorFeeder : public BlobFeederBase { const auto npy_type = PyArray_TYPE(array); const TypeMeta& meta = NumpyTypeToCaffe(npy_type); CAFFE_ENFORCE( - meta.id() != CaffeTypeId::uninitialized(), + meta.id() != TypeIdentifier::uninitialized(), "This numpy data type is not supported: ", PyArray_TYPE(array), "."); diff --git a/caffe2/python/pybind_state_dlpack.cc b/caffe2/python/pybind_state_dlpack.cc index 1fea0a73c86f2..abfe6bb343d26 100644 --- a/caffe2/python/pybind_state_dlpack.cc +++ b/caffe2/python/pybind_state_dlpack.cc @@ -15,7 +15,7 @@ const DLDeviceType* CaffeToDLDeviceType(int device_type) { } const DLDataType* CaffeToDLType(const TypeMeta& meta) { - static std::map dl_type_map{ + static std::map dl_type_map{ {TypeMeta::Id(), DLDataType{0, 8, 1}}, {TypeMeta::Id(), DLDataType{0, 16, 1}}, {TypeMeta::Id(), DLDataType{0, 32, 1}}, diff --git a/caffe2/python/pybind_state_dlpack.h b/caffe2/python/pybind_state_dlpack.h index 37bf82e90bc30..45c074f929f4f 100644 --- a/caffe2/python/pybind_state_dlpack.h +++ b/caffe2/python/pybind_state_dlpack.h @@ -39,7 +39,7 @@ class DLPackWrapper { if (tensor->size() <= 0) { tensor->Resize(0); } - if (tensor->meta().id() == CaffeTypeId::uninitialized()) { + if (tensor->meta().id() == TypeIdentifier::uninitialized()) { // treat uninitialized tensor as float tensor tensor->template mutable_data(); } diff --git a/caffe2/python/pybind_state_ideep.cc b/caffe2/python/pybind_state_ideep.cc index 8b4b98d87714f..668c812cd8e1a 100644 --- a/caffe2/python/pybind_state_ideep.cc +++ b/caffe2/python/pybind_state_ideep.cc @@ -116,7 +116,7 @@ class IDeepFeeder : public BlobFeederBase { const auto npy_type = PyArray_TYPE(array); const TypeMeta& meta = NumpyTypeToCaffe(npy_type); CAFFE_ENFORCE( - meta.id() != CaffeTypeId::uninitialized(), + meta.id() != TypeIdentifier::uninitialized(), "This numpy data type is not supported: ", PyArray_TYPE(array), "."); diff --git a/caffe2/sgd/lars_op.cc b/caffe2/sgd/lars_op.cc index 3e013a943464b..9ab418774e346 100644 --- a/caffe2/sgd/lars_op.cc +++ b/caffe2/sgd/lars_op.cc @@ -10,45 +10,55 @@ void LarsOp::Compute( TIndex N, const float* X_data, const float* dX_data, + const float* wd, + const float* trust, + const float* lr_max, float offset, - float* lr_rescale_data) { - *lr_rescale_data = 1.0; - + float lr_min, + float* lr_rescaled) { + float val = 1.0; float X_norm = sqrtf((ConstEigenVectorMap(X_data, N).array()).square().sum()); - if (X_norm > 0) { float dX_norm = sqrtf((ConstEigenVectorMap(dX_data, N).array()).square().sum()); - *lr_rescale_data /= (dX_norm / X_norm + offset); + val = (*trust) / (dX_norm / X_norm + (*wd) + offset); } + val = fmin(val, *lr_max); + val = fmax(val, lr_min); + *lr_rescaled = val; } REGISTER_CPU_OPERATOR(Lars, LarsOp); OPERATOR_SCHEMA(Lars) - .NumInputs(2) + .NumInputs(5) .NumOutputs(1) .SetDoc(R"DOC( -Implement Layer-wise Adaptive Rate Scaling (LARS) as in -https://arxiv.org/abs/1708.03888. Without weight decay, given a global -learning rate lr, parameter tensor X and its gradient dX, the local learning -rate for X will be +Implement Layer-wise Adaptive Rate Scaling (LARS) with clipping. Before adding weight +decay, given a parameter tensor X and its gradient dX, the local learning rate +for X will be - local_lr = lr * norm(X) / ( norm(dX) + offset * norm(X) ) +local_lr = trust * norm(X) / ( norm(dX) + wd * norm(X) + offset * norm(X) ) - = lr / ( norm(dX) / norm(X) + offset ), + = trust / ( norm(dX) / norm(X) + wd + offset ), -where offset is a preset hyper-parameter to avoid numerical issue. -In this implementation, we uses l2 norm and output the rescaling factor +where offset is a preset hyper-parameter to avoid numerical issue and trust +indicates how much we trust the layer to change its parameters during one update. +In this implementation, we uses l2 norm and the computed local learning rate is +clipped based on the upper bound lr_max and the lower bound lr_min: - 1 / ( norm(dX) / norm(X) + offset ). +local_lr = min(local_lr, lr_max) and local_lr = max(local_lr, lr_min) )DOC") .Input(0, "X", "Parameter tensor") .Input(1, "dX", "Gradient tensor") - .Output(0, "lr_rescale", "Local learning rate rescaling factor") - .Arg("offset", "rescaling offset parameter"); + .Input(2, "wd", "Weight decay") + .Input(3, "trust", "Trust") + .Input(4, "lr_max", "Upper bound of learning rate") + .Output(0, "lr_rescaled", "Rescaled local learning rate") + .Arg("offset", "rescaling offset parameter") + .Arg("lr_min", "minimum learning rate for clipping"); SHOULD_NOT_DO_GRADIENT(Lars); } // namespace caffe2 diff --git a/caffe2/sgd/lars_op.h b/caffe2/sgd/lars_op.h index f4eca35915ff7..0be04476fe2a2 100644 --- a/caffe2/sgd/lars_op.h +++ b/caffe2/sgd/lars_op.h @@ -13,7 +13,8 @@ class LarsOp final : public Operator { USE_OPERATOR_CONTEXT_FUNCTIONS; LarsOp(const OperatorDef& operator_def, Workspace* ws) : Operator(operator_def, ws), - offset_(OperatorBase::GetSingleArgument("offset", 0.5)) {} + offset_(OperatorBase::GetSingleArgument("offset", 0.5)), + lr_min_(OperatorBase::GetSingleArgument("lr_min", 0.02)) {} bool RunOnDevice() override { auto& X = Input(0); @@ -21,16 +22,24 @@ class LarsOp final : public Operator { CAFFE_ENFORCE( dX.size() == X.size(), "Gradient size doesn't match parameter size."); CAFFE_ENFORCE_GE(offset_, 0); + CAFFE_ENFORCE_GE(lr_min_, 0); - auto* lr_rescale = Output(0); - lr_rescale->Resize(vector{1}); + auto& wd = Input(2); + auto& trust = Input(3); + auto& lr_max = Input(4); + auto* lr_rescaled = Output(0); + lr_rescaled->Resize(vector{1}); Compute( dX.size(), X.template data(), dX.template data(), + wd.template data(), + trust.template data(), + lr_max.template data(), offset_, - lr_rescale->template mutable_data()); + lr_min_, + lr_rescaled->template mutable_data()); return true; } @@ -40,10 +49,15 @@ class LarsOp final : public Operator { TIndex N, const T* X_data, const T* dX_data, + const T* wd, + const T* trust, + const T* lr_max, T offset, - T* lr_rescale_data); + T lr_min, + T* lr_rescaled); T offset_; + T lr_min_; }; } // namespace caffe2 diff --git a/docs/requirements.txt b/docs/requirements.txt index 0c3ee1922efaa..159253ca0a6e3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ sphinx -e git://github.com/snide/sphinx_rtd_theme.git#egg=sphinx_rtd_theme +sphinxcontrib.katex diff --git a/docs/source/conf.py b/docs/source/conf.py index 1eaaa3b9086d9..1b4d8d6ff8319 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -19,7 +19,10 @@ # import os # import sys -# sys.path.insert(0, os.path.abspath('.')) + +# source code directory, relative to this file, for sphinx-autobuild +# sys.path.insert(0, os.path.abspath('../..')) + import torch try: import torchvision @@ -47,11 +50,40 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', 'sphinx.ext.napoleon', 'sphinx.ext.viewcode', + 'sphinxcontrib.katex', ] +# katex (mathjax replacement) macros +# +# + +katex_macros = r''' +"\\op": "\\operatorname{{#1}}", +"\\i": "\\mathrm{i}", +"\\e": "\\mathrm{e}^{#1}", +"\\w": "\\omega", +"\\vec": "\\mathbf{#1}", +"\\x": "\\vec{x}", +"\\d": "\\operatorname{d}\\!{}", +"\\dirac": "\\operatorname{\\delta}\\left(#1\\right)", +"\\scalarprod": "\\left\\langle#1,#2\\right\\rangle", +''' + +# katex options +# +# + +katex_options = r''' +delimiters : [ + {left: "$$", right: "$$", display: true}, + {left: "\\(", right: "\\)", display: true}, + {left: "\\[", right: "\\]", display: true} +], +strict : false +''' + napoleon_use_ivar = True # Add any paths that contain templates here, relative to this directory. @@ -106,14 +138,23 @@ autodoc_inherit_docstrings = False -# -- Options for HTML output ---------------------------------------------- +# -- katex javascript in header +# +# def setup(app): +# app.add_javascript("https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.js") + +# -- Options for HTML output ---------------------------------------------- +# # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # +# +# html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. @@ -135,11 +176,12 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static', '_images'] -# html_style_path = 'css/pytorch_theme.css' +html_style_path = 'css/pytorch_theme.css' html_context = { 'css_files': [ 'https://fonts.googleapis.com/css?family=Lato', - '_static/css/pytorch_theme.css' + '_static/css/pytorch_theme.css', + 'https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css', ], } diff --git a/docs/source/nn.rst b/docs/source/nn.rst index 283409ea3676b..9b3563f83b2d4 100644 --- a/docs/source/nn.rst +++ b/docs/source/nn.rst @@ -1120,7 +1120,7 @@ Linear functions .. autofunction:: linear :hidden:`bilinear` -~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~ .. autofunction:: bilinear diff --git a/setup.py b/setup.py index 2e2ef60fb4131..9691285fde67a 100644 --- a/setup.py +++ b/setup.py @@ -762,6 +762,7 @@ def run(self): "torch/csrc/finalizer.cpp", "torch/csrc/jit/batched/BatchTensor.cpp", "torch/csrc/jit/init.cpp", + "torch/csrc/jit/ivalue.cpp", "torch/csrc/jit/passes/onnx.cpp", "torch/csrc/jit/passes/onnx/fixup_onnx_loop.cpp", "torch/csrc/jit/passes/onnx/peephole.cpp", diff --git a/test/expect/TestScript.test_string_cu.expect b/test/expect/TestScript.test_string_cu.expect new file mode 100644 index 0000000000000..602ed319a1327 --- /dev/null +++ b/test/expect/TestScript.test_string_cu.expect @@ -0,0 +1,7 @@ +graph(%a : Dynamic) { + %1 : string = prim::Constant[string=a\n\tb\n]() + %2 : int = prim::Constant[value=2]() + %3 : string = prim::Constant[string=aa]() + = prim::Print(%a, %1, %2, %3) + return (%a); +} diff --git a/test/expect/TestScript.test_string_print-stdout.expect b/test/expect/TestScript.test_string_print-stdout.expect new file mode 100644 index 0000000000000..19f670510f10d --- /dev/null +++ b/test/expect/TestScript.test_string_print-stdout.expect @@ -0,0 +1,2 @@ +1 +[ Variable[CPULongType]{} ] abcd 2 1.5 diff --git a/test/test_dataloader.py b/test/test_dataloader.py index 7a6bb10f82173..bb61cced71753 100644 --- a/test/test_dataloader.py +++ b/test/test_dataloader.py @@ -532,7 +532,7 @@ def _is_process_alive(pid, pname): @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_main_process_unclean_exit(self): - r'''There might be ConnectionResetError or leaked semaphore warning (due to dirty process exit), \ + '''There might be ConnectionResetError or leaked semaphore warning (due to dirty process exit), \ but they are all safe to ignore''' worker_pids = mp.Array('i', [0] * 4) diff --git a/test/test_jit.py b/test/test_jit.py index b3bbe9892bc7d..43d8b05e1077c 100644 --- a/test/test_jit.py +++ b/test/test_jit.py @@ -442,6 +442,29 @@ def f(x, y): ge = self.checkTrace(f, (x, y)) self.assertExpectedGraph(ge.graph_for(x, y)) + @unittest.skipIf(IS_WINDOWS, "NYI: fuser support for Windows") + @unittest.skipIf(not RUN_CUDA, "fuser requires CUDA") + def test_fusion_rand(self): + class M(torch.jit.ScriptModule): + __constants__ = ['d'] + + def __init__(self): + self.d = torch.device('cuda') + + @torch.jit.script_method + def create(self, x): + return x * x + x + torch.rand_like(x) + + x = torch.zeros([3, 4, 5], dtype=torch.float, device='cuda') + m = M() + out1 = m.create(x) + out2 = m.create(x) + self.assertNotEqual(out1, out2) + self.assertTrue(torch.all(out1 >= 0)) + self.assertTrue(torch.all(out1 < 1)) + self.assertTrue(torch.all(out2 >= 0)) + self.assertTrue(torch.all(out2 < 1)) + @staticmethod def fn_test_comparison_gt_lt(x, y): mask = (x > 0).type_as(x) @@ -1825,6 +1848,34 @@ def foo(a): a = Variable(torch.rand(1)) self.assertEqual(a, cu.foo(a)) + # because the compilation unit ingests python strings + # to use an escape sequence escape the backslash (\\n = \n) + def test_string_cu(self): + cu = torch.jit.CompilationUnit(''' + def foo(a): + print(a, """a\\n\tb\\n""", 2, "a\ +a") + return a + ''') + self.assertExpected(str(cu.foo.graph)) + + def test_string_new_line(self): + with self.assertRaisesRegex(RuntimeError, "expected a valid token*"): + torch.jit.CompilationUnit(''' + def test_while(a): + print(" + a") + return a + ''') + + def test_string_single_escape(self): + with self.assertRaisesRegex(RuntimeError, "expected a valid token*"): + torch.jit.CompilationUnit(''' + def test_while(a): + print("\\") + return a + ''') + def test_script_annotation(self): @torch.jit.script def foo(a): @@ -2113,6 +2164,14 @@ def fn(x, y, z): def _make_scalar_vars(self, arr, dtype): return [torch.tensor(val, dtype=dtype) for val in arr] + def test_string_print(self): + def func(a): + print(a, "a" 'b' '''c''' """d""", 2, 1.5) + return a + + inputs = self._make_scalar_vars([1], torch.int64) + self.checkScript(func, inputs, capture_output=True) + def test_while(self): def func(a, b, max): while a < max: @@ -3374,7 +3433,8 @@ def fn(x : torch.Tensor, y : Tensor, z) -> Tuple[Tensor, Tensor, Tensor]: f.write(code) fn = get_fn('test_type_annotation_py3', script_path) - with self.assertRaisesRegex(RuntimeError, r"expected Tensor, but got"): + with self.assertRaisesRegex(RuntimeError, r"expected a value of type Tensor for argument" + r" '0' but found \(Tensor, Tensor\)"): @torch.jit.script def bad_fn(x): x, y = fn((x, x), x, x) @@ -3433,13 +3493,13 @@ def method(self, x): y = self.baz(x) return x - with self.assertRaisesRegex(RuntimeError, "incorrect number of arguments: expected 1, but got 2"): + with self.assertRaisesRegex(RuntimeError, "expected at most 1 arguments but found 2"): ModuleTooMany() - with self.assertRaisesRegex(RuntimeError, "incorrect number of arguments: expected 2, but got 1"): + with self.assertRaisesRegex(RuntimeError, "argument 1 not provided"): ModuleTooFew() with self.assertRaisesRegex(RuntimeError, "need 3 values .* found only 2"): ModuleTooManyAssign() - with self.assertRaisesRegex(RuntimeError, "incorrect number of arguments: expected 2, but got 1"): + with self.assertRaisesRegex(RuntimeError, "argument 1 not provided."): ModuleDefault() def test_script_define_order(self): @@ -4244,18 +4304,6 @@ def test_wrong_use_as_callable(self): def wrong_use_as_callable(x): return x(3, 4, 5) - def test_wrong_python_kwarg_call(self): - with self.assertRaisesRegex(RuntimeError, 'keyword arguments in Python calls aren\'t supported'): - # NB: the only way I could get to this code path is if I made the - # python function have 0 inputs, since we interpret all the inputs to - # the function as inputs (including those with defaults) and not kwargs - def test_fn(): - return 3 - - @torch.jit.script - def wrong_python_kwarg_call(self): - return test_fn(attr=6) - def test_python_val_doesnt_have_attr(self): with self.assertRaisesRegex(RuntimeError, 'object has no attribute abcd'): def test_fn(): @@ -4266,7 +4314,7 @@ def python_val_doesnt_have_attr(): return test_fn.abcd def test_wrong_module_attr_lookup(self): - with self.assertRaisesRegex(RuntimeError, 'unsupported attribute lookup on'): + with self.assertRaisesRegex(RuntimeError, 'python value of type \'type\' cannot be used as a value:'): import io @torch.jit.script @@ -4274,7 +4322,7 @@ def wrong_module_attr_lookup(): return io.BytesIO def test_wrong_method_call_inputs(self): - with self.assertRaisesRegex(RuntimeError, 'argument \'y\' not provided'): + with self.assertRaisesRegex(RuntimeError, 'argument y not provided'): class SomeModule(torch.jit.ScriptModule): @torch.jit.script_method @@ -4849,7 +4897,8 @@ def to_inline(x, y): def some_func(x): return to_inline((x, x), x) - some_func(torch.rand(3, 4)) + x = torch.rand(3, 4) + self.assertEqual(some_func(x), x) def test_file_format_serialization(self): import tempfile diff --git a/test/test_utils.py b/test/test_utils.py index 3fcfc23935bee..c6559fe68fae8 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -616,6 +616,7 @@ def test_bottleneck_cpu_only(self): self._check_cuda(out) @unittest.skipIf(not HAS_CUDA, 'No CUDA') + @unittest.skipIf(TEST_WITH_ROCM, "test doesn't currently work on the ROCm stack") def test_bottleneck_cuda(self): rc, out, err = self._run_bottleneck('bottleneck/test_cuda.py') self.assertEqual(rc, 0, 'Run failed with\n{}'.format(err)) diff --git a/third_party/onnx b/third_party/onnx index 32ac71b1b9c1b..2a3a226a96b3d 160000 --- a/third_party/onnx +++ b/third_party/onnx @@ -1 +1 @@ -Subproject commit 32ac71b1b9c1bd7f196eed3b311734ec6ab3c367 +Subproject commit 2a3a226a96b3dc9e50e739490dd38eaae4f2d8f3 diff --git a/tools/jit/gen_jit_dispatch.py b/tools/jit/gen_jit_dispatch.py index 5a76d447ad249..d337143dd8b09 100644 --- a/tools/jit/gen_jit_dispatch.py +++ b/tools/jit/gen_jit_dispatch.py @@ -191,6 +191,7 @@ def emit_decl_variant(decl): kw_assignments = [] arguments = [] num_inputs = len(decl['arguments']) + op_capture = '' real_inputs = 0 for arg in decl['arguments']: @@ -208,7 +209,8 @@ def emit_decl_variant(decl): constructor = CONSTRUCTOR.substitute(name=decl['name'], call=call, kw_assignments=kw_assignments, - num_inputs=num_inputs) + num_inputs=num_inputs, + op_capture=op_capture) return constructor # This function declares an order on declarations. This is necessary because diff --git a/torch/CMakeLists.txt b/torch/CMakeLists.txt index 057bf6efeac3d..b41b7898e267b 100644 --- a/torch/CMakeLists.txt +++ b/torch/CMakeLists.txt @@ -46,6 +46,13 @@ if("${isSystemDir}" STREQUAL "-1") list(APPEND CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") endif() +# Get the correct Python executable +if (DEFINED ENV{PYTORCH_PYTHON}) + message(STATUS "Using python found in $ENV{PYTORCH_PYTHON}") + set(PYCMD "$ENV{PYTORCH_PYTHON}") +else() + SET(PYCMD "python") +endif() # Generate files set(TOOLS_PATH "${TORCH_SRC_DIR}/../tools") @@ -79,7 +86,7 @@ add_custom_command( "${TORCH_SRC_DIR}/csrc/jit/generated/register_aten_ops.cpp" "${TORCH_SRC_DIR}/csrc/jit/generated/aten_interned_strings.h" COMMAND - python tools/setup_helpers/generate_code.py + ${PYCMD} tools/setup_helpers/generate_code.py --declarations-path "${CMAKE_BINARY_DIR}/aten/src/ATen/Declarations.yaml" --nn-path "aten/src/" DEPENDS @@ -135,6 +142,7 @@ set(TORCH_SRCS ${TORCH_SRC_DIR}/csrc/jit/interpreter.cpp ${TORCH_SRC_DIR}/csrc/jit/constants.cpp ${TORCH_SRC_DIR}/csrc/jit/ir.cpp + ${TORCH_SRC_DIR}/csrc/jit/ivalue.cpp ${TORCH_SRC_DIR}/csrc/jit/operator.cpp ${TORCH_SRC_DIR}/csrc/jit/operator.cpp ${TORCH_SRC_DIR}/csrc/jit/passes/batch_mm.cpp diff --git a/torch/_torch_docs.py b/torch/_torch_docs.py index fad659a4c7145..91cfaff303c0b 100644 --- a/torch/_torch_docs.py +++ b/torch/_torch_docs.py @@ -5715,7 +5715,7 @@ def parse_kwargs(desc): window trims off the last duplicate value from the symmetric window and is ready to be used as a periodic window with functions like :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in -above formula is in fact :math:`\text{window_length} + 1`. Also, we always have +above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have ``torch.hann_window(L, periodic=True)`` equal to ``torch.hann_window(L + 1, periodic=False)[:-1])``. @@ -5733,7 +5733,7 @@ def parse_kwargs(desc): {requires_grad} Returns: - Tensor: A 1-D tensor of size :math:`(\text{{window_length}},)` containing the window + Tensor: A 1-D tensor of size :math:`(\text{{window\_length}},)` containing the window """.format(**factory_common_args)) @@ -5755,7 +5755,7 @@ def parse_kwargs(desc): window trims off the last duplicate value from the symmetric window and is ready to be used as a periodic window with functions like :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in -above formula is in fact :math:`\text{window_length} + 1`. Also, we always have +above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have ``torch.hamming_window(L, periodic=True)`` equal to ``torch.hamming_window(L + 1, periodic=False)[:-1])``. @@ -5776,7 +5776,7 @@ def parse_kwargs(desc): {requires_grad} Returns: - Tensor: A 1-D tensor of size :math:`(\text{{window_length}},)` containing the window + Tensor: A 1-D tensor of size :math:`(\text{{window\_length}},)` containing the window """.format(**factory_common_args)) @@ -5801,7 +5801,7 @@ def parse_kwargs(desc): window trims off the last duplicate value from the symmetric window and is ready to be used as a periodic window with functions like :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in -above formula is in fact :math:`\text{window_length} + 1`. Also, we always have +above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have ``torch.bartlett_window(L, periodic=True)`` equal to ``torch.bartlett_window(L + 1, periodic=False)[:-1])``. @@ -5819,7 +5819,7 @@ def parse_kwargs(desc): {requires_grad} Returns: - Tensor: A 1-D tensor of size :math:`(\text{{window_length}},)` containing the window + Tensor: A 1-D tensor of size :math:`(\text{{window\_length}},)` containing the window """.format(**factory_common_args)) @@ -5841,7 +5841,7 @@ def parse_kwargs(desc): window trims off the last duplicate value from the symmetric window and is ready to be used as a periodic window with functions like :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in -above formula is in fact :math:`\text{window_length} + 1`. Also, we always have +above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have ``torch.blackman_window(L, periodic=True)`` equal to ``torch.blackman_window(L + 1, periodic=False)[:-1])``. @@ -5859,7 +5859,7 @@ def parse_kwargs(desc): {requires_grad} Returns: - Tensor: A 1-D tensor of size :math:`(\text{{window_length}},)` containing the window + Tensor: A 1-D tensor of size :math:`(\text{{window\_length}},)` containing the window """.format(**factory_common_args)) diff --git a/torch/csrc/DynamicTypes.cpp b/torch/csrc/DynamicTypes.cpp index 56be1f731572d..a83a4aa291f35 100644 --- a/torch/csrc/DynamicTypes.cpp +++ b/torch/csrc/DynamicTypes.cpp @@ -69,7 +69,9 @@ at::Type* get_type(const std::string& name, bool is_cuda, bool is_sparse) { PyTypeObject* getPyTypeObject(const at::Storage& storage) { - auto it = attype_to_py_storage_type.find(&storage.pImpl()->type()); + auto attype = at::globalContext().getTypeOpt( + at::detail::get_backend(storage.pImpl()), storage.pImpl()->scalar_type); + auto it = attype_to_py_storage_type.find(attype); if (it != attype_to_py_storage_type.end()) { return it->second; } diff --git a/torch/csrc/autograd/aten_variable_hooks.cpp b/torch/csrc/autograd/aten_variable_hooks.cpp index 2f3899e4f8b59..7a2c3974c2227 100644 --- a/torch/csrc/autograd/aten_variable_hooks.cpp +++ b/torch/csrc/autograd/aten_variable_hooks.cpp @@ -6,7 +6,6 @@ namespace torch { namespace autograd { struct VariableHooks : public at::VariableHooksInterface { VariableHooks(at::VariableHooksArgs) {} void registerVariableTypeFor(at::Context*, at::Backend, at::ScalarType) const override; - at::Type& getVariableType(const at::Type&) const override; }; // Sigh, the registry doesn't support namespaces :( @@ -21,8 +20,4 @@ void VariableHooks::registerVariableTypeFor(at::Context* context, at::Backend ba register_variable_type_for(baseType); } -at::Type& VariableHooks::getVariableType(const at::Type& baseType) const { - return *VariableType::getType(baseType); -} - }} // torch::autograd diff --git a/torch/csrc/autograd/variable.cpp b/torch/csrc/autograd/variable.cpp index 30aded0a85e73..966abb523a7c8 100644 --- a/torch/csrc/autograd/variable.cpp +++ b/torch/csrc/autograd/variable.cpp @@ -22,7 +22,7 @@ namespace torch { namespace autograd { Variable::Impl::Impl(at::Tensor data, bool requires_grad, Edge gradient_edge) - : TensorImpl(data.type().backend(), data.type().scalarType(), nullptr, /* is variable */ true), + : TensorImpl(VariableType::getType(data), nullptr), data_(std::move(data)), grad_fn_(std::move(gradient_edge.function)), requires_grad_(false), @@ -118,9 +118,7 @@ void Variable::Impl::backward( void Variable::Impl::set_data(Tensor new_data) { if (new_data.type() != data_.type()) { - scalar_type_ = new_data.type().scalarType(); - backend_ = new_data.type().backend(); - is_variable_ = true; + type_ = VariableType::getType(new_data.type()); // Clear grad_accumulator if it exists, since it stores the old type info. grad_accumulator_.reset(); } diff --git a/torch/csrc/autograd/variable.h b/torch/csrc/autograd/variable.h index d46008bbdd10b..633b8028f765d 100644 --- a/torch/csrc/autograd/variable.h +++ b/torch/csrc/autograd/variable.h @@ -327,6 +327,9 @@ struct Variable::Impl : public at::TensorImpl { /// Reset all expensive fields to free up resources void release_resources() override; + // Make this field public so we can access it from `Variable`. + using at::TensorImpl::type_; + std::string name; at::Tensor data_; diff --git a/torch/csrc/jit/constants.cpp b/torch/csrc/jit/constants.cpp index 47e593bbb125e..698153aad27fa 100644 --- a/torch/csrc/jit/constants.cpp +++ b/torch/csrc/jit/constants.cpp @@ -29,6 +29,9 @@ Value* insertConstant( return autograd::Variable(t).data(); })); n->output()->setType(ListType::ofTensors()); + } else if(val.isString()) { + n->s_(attr::string, val.toString()->string()); + n->output()->setType(StringType::get()); } else { throw std::runtime_error("Unsupported value kind: " + val.tagKind()); } @@ -79,6 +82,12 @@ RegisterOperators reg({ push(stack, ts); return 0; }; + } else if (type == StringType::get()) { + auto s = node->s(attr::string); + return [s](Stack& stack) { + push(stack, s); + return 0; + }; } else { std::stringstream ss; ss << "constant literal not supported for: " << type->str(); diff --git a/torch/csrc/jit/fusion_compiler.cpp b/torch/csrc/jit/fusion_compiler.cpp index 22f8b40ba3054..dad50440944e8 100644 --- a/torch/csrc/jit/fusion_compiler.cpp +++ b/torch/csrc/jit/fusion_compiler.cpp @@ -15,6 +15,7 @@ #ifdef USE_CUDA #include "ATen/cuda/CUDAContext.h" #include "THC/THC.h" +#include #include "torch/csrc/cuda/cuda_check.h" #include #include @@ -30,6 +31,10 @@ #include #include +#ifdef USE_CUDA +THCGenerator* THCRandom_getGenerator(THCState* state); +#endif + namespace torch { namespace jit { std::vector TensorDesc::findContiguous( @@ -78,6 +83,7 @@ typedef signed char int8_t; typedef short int int16_t; typedef long long int int64_t; ${HalfHeader} +${RandHeader} #endif typedef ${IndexType} IndexType; template @@ -88,11 +94,113 @@ struct TensorInfo { }; )"); +// We rewrite the code for philox RNG from curand as nvrtc couldn't resolve the +// curand header correctly. +constexpr auto rand_support_literal = R"( + + class Philox { + public: + __device__ inline Philox(unsigned long long seed, + unsigned long long subsequence, + unsigned long long offset) { + key.x = (unsigned int)seed; + key.y = (unsigned int)(seed >> 32); + counter = make_uint4(0, 0, 0, 0); + counter.z = (unsigned int)(subsequence); + counter.w = (unsigned int)(subsequence >> 32); + STATE = 0; + incr_n(offset / 4); + } + + __device__ inline unsigned long operator()() { + if(STATE == 0) { + uint4 counter_ = counter; + uint2 key_ = key; + for(int i = 0; i < 9; i++) { + counter_ = single_round(counter_, key_); + key_.x += (kPhilox10A); key_.y += (kPhilox10B); + } + output = single_round(counter_, key_); + incr(); + } + unsigned long ret; + switch(STATE) { + case 0: ret = output.x; break; + case 1: ret = output.y; break; + case 2: ret = output.z; break; + case 3: ret = output.w; break; + } + STATE = (STATE + 1) % 4; + return ret; + } + + private: + uint4 counter; + uint4 output; + uint2 key; + unsigned int STATE; + __device__ inline void incr_n(unsigned long long n) { + unsigned int nlo = (unsigned int)(n); + unsigned int nhi = (unsigned int)(n >> 32); + counter.x += nlo; + if (counter.x < nlo) + nhi++; + counter.y += nhi; + if (nhi <= counter.y) + return; + if (++counter.z) + return; + ++counter.w; + } + __device__ inline void incr() { + if (++counter.x) + return; + if (++counter.y) + return; + if (++counter.z) + return; + ++counter.w; + } + __device__ unsigned int mulhilo32(unsigned int a, unsigned int b, + unsigned int *result_high) { + *result_high = __umulhi(a, b); + return a*b; + } + + __device__ inline uint4 single_round(uint4 ctr, uint2 key) { + unsigned int hi0; + unsigned int hi1; + unsigned int lo0 = mulhilo32(kPhiloxSA, ctr.x, &hi0); + unsigned int lo1 = mulhilo32(kPhiloxSB, ctr.z, &hi1); + + uint4 ret = {hi1 ^ ctr.y ^ key.x, lo1, hi0 ^ ctr.w ^ key.y, lo0}; + return ret; + } + + static const unsigned long kPhilox10A = 0x9E3779B9; + static const unsigned long kPhilox10B = 0xBB67AE85; + static const unsigned long kPhiloxSA = 0xD2511F53; + static const unsigned long kPhiloxSB = 0xCD9E8D57; + }; + + // Inverse of 2^32. + #define M_RAN_INVM32 2.3283064e-10f + __device__ __inline__ float uniform(unsigned int x) { + return x * M_RAN_INVM32; + } +)"; + +constexpr auto rand_param = ",unsigned long long seed, unsigned long long offset"; +constexpr auto rand_init = R"( + int idx = blockIdx.x*blockDim.x + threadIdx.x; + Philox rnd(seed, idx, offset); +)"; auto cuda_compilation_unit_template = CodeTemplate(R"( ${type_declarations} extern "C" __global__ -void ${kernelName}(IndexType totalElements, ${formals}) { +void ${kernelName}(IndexType totalElements, ${formals} ${RandParam}) { + ${RandInit} for (IndexType linearIndex = blockIdx.x * blockDim.x + threadIdx.x; linearIndex < totalElements; linearIndex += gridDim.x * blockDim.x) { @@ -280,9 +388,10 @@ std::string encodeRHS(Node * n) { {aten::remainder, "remainderf(${0}, ${1})"}, {aten::pow, "powf(${0}, ${1})"}, - // binary with alpha + //alpha {aten::add, "${0} + ${2}*${1}"}, {aten::sub, "(${0} - ${2}*${1})"}, + {aten::rand_like, "uniform(rnd())"}, // simple derivatives {aten::_sigmoid_backward, "${0} * ${1} * (1.f - ${1})"}, @@ -309,10 +418,12 @@ std::string encodeRHS(Node * n) { return format(str, env); } -std::vector emitCompilationUnit(std::ostream & out, - const std::string & name, - AnnotatedGraph & agraph, - bool use_cuda) { +std::pair, bool> emitCompilationUnit( + std::ostream& out, + const std::string& name, + AnnotatedGraph& agraph, + bool use_cuda) { + bool has_random = false; Graph& subgraph = *agraph.graph; TemplateEnv env; env.s("kernelName",name); @@ -385,6 +496,11 @@ std::vector emitCompilationUnit(std::ostream & out, // FusedConcat nodes work by narrowing the output Tensors before the kernel runs if (n->kind() == prim::FusedConcat) continue; + if(n->kind() == aten::rand_like) { + has_random = true; + if(!use_cuda) + throw std::runtime_error("Fusion doesn't support rand on CPU"); + } env.s("node",valueName(n->output())); env.s("rhs", encodeRHS(n)); body << format("auto ${node} = ${rhs};\n",env); @@ -412,6 +528,16 @@ std::vector emitCompilationUnit(std::ostream & out, env.s("HalfHeader", ""); } + if (has_random) { + env.s("RandHeader", rand_support_literal); + env.s("RandParam", rand_param); + env.s("RandInit", rand_init); + } else { + env.s("RandHeader", ""); + env.s("RandParam", ""); + env.s("RandInit", ""); + } + env.s("tensorOffsets",tensorOffsets.str()); env.s("kernelBody",body.str()); env.v("formals",formals); @@ -423,7 +549,7 @@ std::vector emitCompilationUnit(std::ostream & out, out << cpu_compilation_unit_template.format(env); } - return concat_desc; + return std::make_pair(std::move(concat_desc), has_random); } //////////////////////////////////////////////////////////////////////////////// @@ -518,7 +644,7 @@ void CompiledFusionFunction::launch_with_tensors(at::ArrayRef inputs char * buffer_next = buffer.data(); // A vector of arguments to the kernel. It's (numel, *input_descs, *output_descs) std::vector arguments; - arguments.reserve(1 + inputs.size() + flat_outputs_size); + arguments.reserve(3 + inputs.size() + flat_outputs_size); // Asserts that t's dims can be compressed in the same way as in desc // (that's what the kernel assumes), and appends it to the arguments vector. auto addTensorInfo = [&](TensorDesc & desc, const at::Tensor & t) { @@ -555,6 +681,19 @@ void CompiledFusionFunction::launch_with_tensors(at::ArrayRef inputs } } } + + // If the kernel call contains a random op, we need to pass in random seeds as + // well. + #ifdef USE_CUDA + if(has_random && this->backend() == at::kCUDA) { + auto gen_ = THCRandom_getGenerator(at::globalContext().getTHCState()); + uint64_t offset = + gen_->state.philox_seed_offset.fetch_add(this->get_rand_offset(numel)); + arguments.push_back(&gen_->state.initial_seed); + arguments.push_back(&offset); + } + #endif + launch_raw(numel, arguments.data()); } @@ -590,13 +729,15 @@ struct CUDAFusionFunction : public CompiledFusionFunction { checkCUDAVersion(prop); std::stringstream cu; - concat_desc = codegen::emitCompilationUnit(cu, name, agraph, true); + auto ret = codegen::emitCompilationUnit(cu, name, agraph, true); + concat_desc = std::move(ret.first); + has_random = ret.second; compilation_unit = cu.str(); nvrtcProgram program; TORCH_NVRTC_CHECK(nvrtcCreateProgram(&program, compilation_unit.c_str(), NULL, 0, nullptr, nullptr)); std::string compute = "--gpu-architecture=compute_" + std::to_string(prop.major) + std::to_string(prop.minor); - std::vector args = {"--std=c++11", compute.c_str()}; + std::vector args = {"--std=c++11", compute.c_str(), "-default-device"}; nvrtcResult result = nvrtcCompileProgram(program, args.size(), args.data()); if (result == NVRTC_ERROR_COMPILATION) { size_t logsize; @@ -630,8 +771,13 @@ struct CUDAFusionFunction : public CompiledFusionFunction { virtual at::Backend backend() const override { return at::kCUDA; } + virtual uint64_t get_rand_offset(uint32_t numel) override { + int numBlocks = std::min(maxBlocks, ceilDiv(numel, blockSize)); + return 4 * (ceil(numel/(4 * blockSize * numBlocks)) + 1); + } virtual void launch_raw(uint32_t numel, void ** arguments) override { int numBlocks = std::min(maxBlocks, ceilDiv(numel, blockSize)); + //std::cout << "maxBlocks = " << maxBlocks << " needed blocks: " << ceilDiv(numel,blockSize) // << " numblocks = " << numBlocks; @@ -786,7 +932,10 @@ struct CPUFusionFunction : public CompiledFusionFunction { TempFile cpp_file(cpp_template, 4); std::stringstream cu; - concat_desc = codegen::emitCompilationUnit(cu, name, agraph, false); + auto ret = codegen::emitCompilationUnit(cu, name, agraph, false); + concat_desc = std::move(ret.first); + has_random = ret.second; + JIT_ASSERT(!has_random); compilation_unit = cu.str(); cpp_file.write(compilation_unit); cpp_file.sync(); @@ -803,6 +952,9 @@ struct CPUFusionFunction : public CompiledFusionFunction { virtual at::Backend backend() const override { return at::kCPU; } + virtual uint64_t get_rand_offset(uint32_t numel) override { + return numel; + } virtual void launch_raw(uint32_t numel, void ** arguments) override { kernel(numel, arguments); } diff --git a/torch/csrc/jit/fusion_compiler.h b/torch/csrc/jit/fusion_compiler.h index c2f35ee0aa207..99a2303d37af2 100644 --- a/torch/csrc/jit/fusion_compiler.h +++ b/torch/csrc/jit/fusion_compiler.h @@ -5,7 +5,6 @@ #include #include "ATen/ATen.h" - #include #include #include @@ -109,6 +108,9 @@ struct CompiledFusionFunction { // launch_with_tensors handles packing at::Tensors into this arguments array. // CPU code uses the same convension so that launch_with_tensors can be shared. virtual void launch_raw(uint32_t numel, void ** arguments) = 0; + + virtual uint64_t get_rand_offset(uint32_t numel) = 0; + bool has_random; std::string name; // We keep these around for debugging std::string compilation_unit; diff --git a/torch/csrc/jit/interned_strings.h b/torch/csrc/jit/interned_strings.h index c567793552d73..fd6208147dffc 100644 --- a/torch/csrc/jit/interned_strings.h +++ b/torch/csrc/jit/interned_strings.h @@ -90,7 +90,9 @@ _(attr, sizes) \ _(attr, starts) \ _(attr, transA) \ _(attr, transB) \ -_(attr, name) +_(attr, name) \ +_(attr, string) + // 'prim' symbols are synthetic operators that occur only in the IR // and don't have corresponding implementations in ATen. diff --git a/torch/csrc/jit/interpreter.cpp b/torch/csrc/jit/interpreter.cpp index 0c1fe17ade0df..da6f629d629e4 100644 --- a/torch/csrc/jit/interpreter.cpp +++ b/torch/csrc/jit/interpreter.cpp @@ -337,7 +337,7 @@ struct PreprocessGraph { struct ContainerTensor : public at::TensorImpl { public: ContainerTensor() - : TensorImpl(at::Backend::Undefined,at::ScalarType::Undefined, nullptr, /* is_variable */ false) {} + : TensorImpl(&(at::globalContext().getType(at::Backend::Undefined,at::ScalarType::Undefined)), nullptr) {} virtual ~ContainerTensor() = default; virtual at::IntList sizes() const override { diff --git a/torch/csrc/jit/ir.cpp b/torch/csrc/jit/ir.cpp index ede14249c46dc..e273084be642d 100644 --- a/torch/csrc/jit/ir.cpp +++ b/torch/csrc/jit/ir.cpp @@ -82,6 +82,20 @@ void printPrimList(std::ostream & out, const std::vector & items) { } out << "]"; } + +std::string escapeString(std::string s) { + std::vector search = {'\n', '\t', '\v'}; + std::vector replace = {"\\n", "\\t", "\\v"}; + for (size_t i = 0; i < search.size(); i++) { + size_t pos = s.find(search[i]); + while(pos != std::string::npos) { + s.replace(pos, 1, replace[i]); + pos = s.find(search[i], pos + 1); + } + } + return s; +} + void printAttributes(std::ostream & out, const Node * n, bool ignore_subgraph=false) { out << "["; auto names = n->attributeNames(); @@ -110,7 +124,7 @@ void printAttributes(std::ostream & out, const Node * n, bool ignore_subgraph=fa printPrimList(out,n->is(name)); break; case AttributeKind::s: - out << n->s(name); + out << escapeString(n->s(name)); break; case AttributeKind::ss: printPrimList(out,n->ss(name)); diff --git a/torch/csrc/jit/ir.h b/torch/csrc/jit/ir.h index b2caa642b6fe2..959228e154779 100644 --- a/torch/csrc/jit/ir.h +++ b/torch/csrc/jit/ir.h @@ -987,7 +987,6 @@ friend struct Block; Node * createUndefined() { return create(prim::Undefined); } - Node * createFusionGroup(int device) { auto n = create(prim::FusionGroup, 0); n->g_(attr::Subgraph,std::make_shared(scope_root_)); diff --git a/torch/csrc/jit/ivalue.cpp b/torch/csrc/jit/ivalue.cpp new file mode 100644 index 0000000000000..57460bf1aa469 --- /dev/null +++ b/torch/csrc/jit/ivalue.cpp @@ -0,0 +1,20 @@ +#include "torch/csrc/jit/assertions.h" +#include "torch/csrc/jit/ivalue.h" +#include + +#define TORCH_FORALL_TAGS(_) \ + _(None) _(Tensor) _(Double) _(Int) _(Tuple) _(IntList) _(DoubleList) _(String) _(TensorList) + +namespace torch { namespace jit { +std::ostream& operator<<(std::ostream & out, const IValue & v) { + switch(v.tag) { + #define DEFINE_CASE(x) case IValue::Tag::x: return v.format ## x(out); + TORCH_FORALL_TAGS(DEFINE_CASE) + #undef DEFINE_CASE + } + AT_ERROR("Tag not found\n"); +} + +#undef TORCH_FORALL_TAGS + +}} diff --git a/torch/csrc/jit/ivalue.h b/torch/csrc/jit/ivalue.h index 6eef40a032306..0ad0323c1bf34 100644 --- a/torch/csrc/jit/ivalue.h +++ b/torch/csrc/jit/ivalue.h @@ -1,6 +1,7 @@ #pragma once #include "torch/csrc/jit/assertions.h" +#include "torch/csrc/WindowsTorchApiMacro.h" #include @@ -77,6 +78,28 @@ struct Shared { PointerType * pImpl; }; +// string +struct ConstantString : at::Retainable { + private: + ConstantString(const std::string & str) + : str_(str) {} + const std::string str_; + public: + static Shared create(const std::string str_) { + return Shared( + new ConstantString(str_), false); + } + const std::string & string() const { + return str_; + } + operator const std::string & () const { + return string(); + } + TORCH_API std::ostream& operator<<(std::ostream & out) const { + out << string(); + return out; + } +}; template struct ConstantList; @@ -94,7 +117,7 @@ using DoubleList = ConstantList; // retain/release calls. #define TORCH_FORALL_TAGS(_) \ - _(None) _(Tensor) _(Double) _(Int) _(Tuple) _(IntList) _(DoubleList) _(TensorList) + _(None) _(Tensor) _(Double) _(Int) _(Tuple) _(IntList) _(DoubleList) _(String) _(TensorList) struct IValue { IValue() @@ -151,6 +174,11 @@ struct IValue { JIT_ASSERT(isTensor()); return at::Tensor(as_tensor_impl, /*retain=*/true); } + TORCH_API std::ostream& formatTensor(std::ostream& out) const { + JIT_ASSERT(isTensor()); + out << toTensor(); + return out; + } // Tuple IValue(Shared v); @@ -163,6 +191,11 @@ struct IValue { JIT_ASSERT(isTuple()); return toRetainable(); } + TORCH_API std::ostream& formatTuple(std::ostream& out) const { + JIT_ASSERT(isTuple()); + out << "Tuple"; //TODO + return out; + } // Double IValue(double d) @@ -174,6 +207,11 @@ struct IValue { JIT_ASSERT(isDouble()); return as_double; } + TORCH_API std::ostream& formatDouble(std::ostream& out) const { + JIT_ASSERT(isDouble()); + out << as_double; + return out; + } // Int IValue(int64_t i) @@ -193,6 +231,12 @@ struct IValue { JIT_ASSERT(isInt()); return as_int; } + TORCH_API std::ostream& formatInt(std::ostream& out) const { + JIT_ASSERT(isInt()); + out << as_int; + return out; + } + // IntList IValue(Shared v); @@ -208,9 +252,32 @@ struct IValue { JIT_ASSERT(isIntList()); return toRetainable(); } + TORCH_API std::ostream& formatIntList(std::ostream& out) const { + JIT_ASSERT(isIntList()); + out << "Int List"; //FIXME @eellison toRetainable(); + return out; + } std::vector copyToIntList() const; + // ConstantString + IValue(Shared v); + IValue(const std::string& v); + bool isString() const { return Tag::String == tag; } + Shared toString() && { + JIT_ASSERT(isString()); + return moveToRetainable(); + } + Shared toString() const & { + JIT_ASSERT(isString()); + return toRetainable(); + } + TORCH_API std::ostream& formatString(std::ostream& out) const { + JIT_ASSERT(isString()); + out << toRetainable()->string(); + return out; + } + // DoubleList IValue(Shared v); IValue(std::vector v); @@ -223,6 +290,12 @@ struct IValue { JIT_ASSERT(isDoubleList()); return toRetainable(); } + TORCH_API std::ostream& formatDoubleList(std::ostream& out) const { + JIT_ASSERT(isDoubleList()); + out << "Double List"; //FIXME @eellison toRetainable(); + return out; + } + //TensorList IValue(Shared v); @@ -236,11 +309,20 @@ struct IValue { JIT_ASSERT(isTensorList()); return toRetainable(); } + TORCH_API std::ostream& formatTensorList(std::ostream& out) const { + JIT_ASSERT(isTensorList()); + out << "Tensor List"; //FIXME @eellison toRetainable(); + return out; + } // None bool isNone() { return Tag::None == tag; } + std::ostream& formatNone(std::ostream& out) const { + out << "None"; + return out; + } // Scalar, which gets encoded as either an Int or a Double IValue(at::Scalar s) @@ -264,7 +346,7 @@ struct IValue { } // for debugging - std::string tagKind() { + std::string tagKind() const { switch(tag) { #define DEFINE_CASE(x) case Tag::x: return #x; TORCH_FORALL_TAGS(DEFINE_CASE) @@ -287,6 +369,8 @@ struct IValue { template T to() const &; + TORCH_API friend std::ostream& operator<<(std::ostream & out, const IValue & v); + private: template Shared moveToRetainable() { @@ -339,10 +423,12 @@ DEFINE_TO(double, toDouble) DEFINE_TO(int64_t, toInt) DEFINE_TO(Shared, toDoubleList) DEFINE_TO(Shared, toIntList) +DEFINE_TO(Shared, toString) DEFINE_TO(at::Scalar, toScalar) DEFINE_TO(bool, toInt) DEFINE_TO(std::vector, copyToIntList) + #undef DEFINE_TO // non-mutable list @@ -365,6 +451,7 @@ struct ConstantList : at::Retainable { } }; + inline IValue::IValue(Shared v) : tag(Tag::Tuple), retainable(true) { as_retainable = v.detach(); @@ -377,6 +464,13 @@ inline IValue::IValue(Shared v) inline IValue::IValue(std::vector v) : IValue(IntList::create(std::move(v))) {} +inline IValue::IValue(Shared v) +: tag(Tag::String), retainable(true) { + as_retainable = v.detach(); +} +inline IValue::IValue(const std::string& v) +: IValue(ConstantString::create(v)) {} + inline IValue::IValue(Shared v) : tag(Tag::DoubleList), retainable(true) { as_retainable = v.detach(); @@ -395,4 +489,5 @@ inline std::vector IValue::copyToIntList() const { return toIntList()->elements().vec(); } + }} diff --git a/torch/csrc/jit/passes/graph_fuser.cpp b/torch/csrc/jit/passes/graph_fuser.cpp index cc8dcb8926dee..071ca6d57cd2d 100644 --- a/torch/csrc/jit/passes/graph_fuser.cpp +++ b/torch/csrc/jit/passes/graph_fuser.cpp @@ -75,6 +75,7 @@ std::unordered_set simple_mappable = { // TODO support those //aten::clamp, //aten::lerp, + aten::rand_like, }; bool isSimpleMap(Node *node) { diff --git a/torch/csrc/jit/pybind_utils.h b/torch/csrc/jit/pybind_utils.h index 0598e651d3243..94c4d53be24cb 100644 --- a/torch/csrc/jit/pybind_utils.h +++ b/torch/csrc/jit/pybind_utils.h @@ -20,9 +20,10 @@ inline Stack createStack(const py::tuple& tuple, at::ArrayRef inputs, si return py::cast(obj); case TypeKind::NoneType: return {}; + case TypeKind::StringType: case TypeKind::ListType: case TypeKind::TupleType: - throw std::runtime_error("Lists and tuples are not supported yet"); + throw std::runtime_error("Lists tuples and strings are not supported yet"); case TypeKind::NumberType: throw std::runtime_error("Insufficient type information to convert input"); } @@ -52,9 +53,10 @@ inline py::object wrapStack(Stack&& outputs, at::ArrayRef output_vals) { return py::cast(ivalue.toInt()); case TypeKind::NoneType: return py::none(); + case TypeKind::StringType: case TypeKind::ListType: case TypeKind::TupleType: - throw std::runtime_error("Lists and tuples are not supported yet"); + throw std::runtime_error("Lists tuples and strings are not supported yet"); case TypeKind::NumberType: throw std::runtime_error("Insufficient type information to convert input"); } diff --git a/torch/csrc/jit/register_prim_ops.cpp b/torch/csrc/jit/register_prim_ops.cpp index f2b8ea18a2be2..82ba798bfc753 100644 --- a/torch/csrc/jit/register_prim_ops.cpp +++ b/torch/csrc/jit/register_prim_ops.cpp @@ -90,33 +90,17 @@ RegisterOperators reg({ return 0; }; }), - Operator( - prim::None, - [](Node* node) { - return [](Stack& stack) { - stack.push_back(IValue()); - return 0; - }; - }), Operator( prim::Print, [](Node* node) { size_t num_inputs = node->inputs().size(); return [num_inputs](Stack& stack) { bool first = true; - for (const IValue& i_ : last(stack, num_inputs)) { - auto i = i_.toTensor(); + for (const IValue& i : last(stack, num_inputs)) { if (!first) std::cout << " "; first = false; - if (auto tensor_impl = dynamic_cast(i.get())) { - std::cout << at::Tensor(tensor_impl, true); - } else if (!i.defined()) { - std::cout << ""; - } else { - auto& r = *i.get(); - std::cout << "<" << typeid(r).name() << " at " << i << ">"; - } + std::cout << i; } drop(stack, num_inputs); std::cout << std::endl; diff --git a/torch/csrc/jit/script/compiler.cpp b/torch/csrc/jit/script/compiler.cpp index 4f27cb25b53cb..63acb23fba044 100644 --- a/torch/csrc/jit/script/compiler.cpp +++ b/torch/csrc/jit/script/compiler.cpp @@ -46,9 +46,19 @@ struct PrintValue : public SugaredValue { auto& g = *m.graph(); if (!attributes.empty()) throw ErrorReport(loc) << "print doesn't accept any keyword arguments"; - auto values = toValues(inputs); - ensureTensors(loc, values); - g.insertNode(g.create(prim::Print, values, 0) + + //temporary hack to allow print statements to work in python 2, where + //print(a, b) is treated as a (a, b) tuple input. + + std::vector lowered_inputs = toValues(inputs); + if(lowered_inputs.size() == 1 && lowered_inputs.at(0)->node()->kind() == prim::TupleConstruct) { + auto input = lowered_inputs[0]; + for(size_t j = 0; j < input->node()->inputs().size(); ++j) { + lowered_inputs.insert(lowered_inputs.begin() + 1 + j, input->node()->inputs().at(j)); + } + lowered_inputs.erase(lowered_inputs.begin()); + } + g.insertNode(g.create(prim::Print, lowered_inputs, 0) ->setSourceLocation(std::make_shared(loc))); return std::make_shared(); } @@ -189,8 +199,9 @@ struct Environment { SugaredValuePtr createCapturedInputIfNeeded(const SourceRange& loc, std::string ident) { auto in_frame = findInThisFrame(ident); - if (in_frame) + if (in_frame) { return in_frame; + } // recursively handles the case where parent blocks are also loops auto from_parent = next ? next->createCapturedInputIfNeeded(loc, ident) : nullptr; @@ -266,7 +277,7 @@ struct Environment { auto retval = createCapturedInputIfNeeded(range, ident); if(!retval) { - retval = resolver(ident); + retval = resolver(ident, method, range); } if(!retval) { @@ -417,7 +428,7 @@ at::optional> tryMatchSchema( return at::nullopt; } if(positional_inputs[*idx]) { - err() << "argument '" << nv.name << "' specified twice \n" << nv.loc; + err() << "argument " << nv.name << " specified twice \n" << nv.loc; return at::nullopt; } positional_inputs[*idx] = nv; @@ -428,7 +439,7 @@ at::optional> tryMatchSchema( continue; auto default_value = schema.arguments[i].default_value; if(!default_value) { - err() << "argument '" << schema.arguments[i].name << "' not provided.\n" << loc; + err() << "argument " << schema.arguments[i].name << " not provided.\n" << loc; return at::nullopt; } positional_inputs[i] = NamedValue( @@ -1363,6 +1374,9 @@ struct to_ir { case TK_IF_EXPR: { return emitTernaryIf(TernaryIf(tree)); } break; + case TK_STRINGLITERAL: { + return emitStringLiteral(StringLiteral(tree)); + } break; case TK_LIST_LITERAL: { auto ll = ListLiteral(tree); auto values = getValues(ll.inputs(), /*maybe_unpack=*/true, identity); @@ -1393,6 +1407,10 @@ struct to_ir { return insertConstant(*graph, c.asIntegral(), c.range()); } + Value* emitStringLiteral(const StringLiteral& c) { + return insertConstant(*graph, c.text(), c.range()); + } + // Desugars slice syntactic sugar tensor[begin:end] -> tensor.slice(begin, // end). Value* emitSlice( diff --git a/torch/csrc/jit/script/compiler.h b/torch/csrc/jit/script/compiler.h index 3c4dcb07a248e..c5175815144e1 100644 --- a/torch/csrc/jit/script/compiler.h +++ b/torch/csrc/jit/script/compiler.h @@ -131,7 +131,8 @@ struct TORCH_API BuiltinFunction : public SugaredValue { size_t n_binders) override; }; -using Resolver = std::function(const std::string& name)>; +using Resolver = std::function(const std::string& name, Method& m, const SourceRange& loc)>; TORCH_API void defineMethodsInModule( Module & m, const std::vector& definitions, diff --git a/torch/csrc/jit/script/init.cpp b/torch/csrc/jit/script/init.cpp index 39bb51ed89ca5..b91d348ed627d 100644 --- a/torch/csrc/jit/script/init.cpp +++ b/torch/csrc/jit/script/init.cpp @@ -45,18 +45,40 @@ inline std::shared_ptr toSimple(Value* v) { return std::make_shared(v); } +// NB: This should be the single entry-point for instantiating a SugaredValue +// from a Python object. If you are adding support for converting a new Python +// type, *add it in this function's implementation*. +std::shared_ptr toSugaredValue( + py::object obj, + Method& m, + SourceRange loc, + bool is_constant = false, + bool is_submodule = false); + struct VISIBILITY_HIDDEN PythonValue : public SugaredValue { PythonValue(py::object self) : self(std::move(self)) {} - std::tuple, std::vector> getArgumentTypes(const size_t n_args, const size_t n_binders) { + FunctionSchema getSchema(const size_t n_args, const size_t n_binders) { auto annotations = py::module::import("torch.jit.annotations"); auto signature = annotations.attr("get_signature")(self); + std::vector args, rets; // We may mutate this if we can determine the number of args from Python // introspection. size_t actual_n_args = n_args; if (!signature.is_none()) { - return py::cast, std::vector>>(signature); + std::vector arg_types, ret_types; + std::tie(arg_types, ret_types) = py::cast, std::vector>>(signature); + args.reserve(arg_types.size()); + size_t idx = 0; // Fake argument names by putting in the index + for (auto &arg_type : arg_types) { + args.push_back(Argument(std::to_string(idx++), std::move(arg_type), {}, {}, false)); + } + rets.reserve(ret_types.size()); + idx = 0; + for (auto &ret_type : ret_types) { + rets.push_back(Argument(std::to_string(idx++), std::move(ret_type), {}, {}, false)); + } } else { // Create a default signature using what information we have @@ -71,62 +93,36 @@ struct VISIBILITY_HIDDEN PythonValue : public SugaredValue { } // Construct the default signature: all arguments and returns will be // DynamicType - return std::make_pair(std::vector(actual_n_args, DynamicType::get()), std::vector(n_binders, DynamicType::get())); + args.reserve(actual_n_args); + for (size_t i=0; i < actual_n_args; ++i) { + args.push_back(Argument(std::to_string(i), DynamicType::get(), {}, {}, false)); + } + rets.reserve(n_binders); + for (size_t i = 0; i < n_binders; ++i) { + rets.push_back(Argument(std::to_string(i), DynamicType::get(), {}, {}, false)); + } } + return FunctionSchema("", std::move(args), std::move(rets)); } // call it like a function, e.g. `outputs = this(inputs)` virtual std::shared_ptr call(SourceRange loc, Method & m, at::ArrayRef inputs_, at::ArrayRef attributes, size_t n_binders) override { auto inputs = toValues(inputs_); - std::vector arguments, returns; - std::tie(arguments, returns) = getArgumentTypes(inputs.size(), n_binders); - - if (arguments.size() != inputs.size()) - throw ErrorReport(loc) << "calling a Python function with an incorrect number " - << "of arguments: expected " << arguments.size() << ", but got " - << inputs.size(); - for (size_t i = 0; i < arguments.size(); ++i) { - if (!inputs[i]->type()->isSubtypeOf(arguments[i])) - throw ErrorReport(loc) << "type mismatch at argument " << i << ": expected " - << arguments[i]->str() << ", but got " << inputs[i]->type()->str(); - } - // We have to do this check here, because implementation of this function is tightly - // coupled with the impl for PythonOp in the interpreter. Right now it assumes that - // all inputs taken from the stack are Tensors, so that's what we have to do. - ensureTensors(loc, inputs); - - if (attributes.size() > 0) - throw ErrorReport(loc) << "keyword arguments in Python calls aren't supported"; - Graph& g = *m.graph(); - - // this python object might be a @trace or @script function/module - // if so, inline the graph rather than calling the python - - if(py::isinstance(self)) { - Module& mod = py::cast(self); - if (Method * forward = mod.find_method("forward")) { - // This code path should only get called for Modules that are really - // wrappers around pure script/traced functions. Modules with parameters - // should be submodules of the caller, and thus will be represented as - // ModuleValue and not go through here. - if (mod.get_parameters().size() != 0) { - throw ErrorReport(loc) << "Attempted to inline a Module with parameters. " - "Stateful modules to be inlined must be submodules of the callee."; - } - std::vector named_inputs; - for (auto inp : inputs) - named_inputs.push_back(NamedValue(loc, "", inp)); - return packOutputs(*m.graph(), m.emit_call_to(loc, *forward, named_inputs, {})); - } - } + auto schema = getSchema(inputs.size(), n_binders); + + std::stringstream failure_messages; + at::optional> all_inputs = + tryMatchSchema(schema, loc, *m.graph(), inputs_, attributes, failure_messages); + if (!all_inputs) + throw ErrorReport(loc) << failure_messages.str(); // Release the function object so we can wrap it in a PythonOp py::object func = self; std::string cconv(inputs.size(), 't'); - Node* new_node = g.insertNode(g.createPythonOp( + Node* new_node = m.graph()->insertNode(m.graph()->createPythonOp( THPObjectPtr(func.release().ptr()), cconv, {})); new_node->setSourceLocation(std::make_shared(loc)); - for(auto i : inputs) + for(auto &i : *all_inputs) new_node->addInput(i); // This is really dumb, but relaxing the constraints on return types would @@ -134,14 +130,14 @@ struct VISIBILITY_HIDDEN PythonValue : public SugaredValue { // Note that this effectively makes the return type of Tuple[Tensor] and Tensor // equivalent, but the PythonOp impl ends with an optional tuple unpack, so we need // to do it. - for (auto & ret_type_elem : returns) { - if (!ret_type_elem->isSubtypeOf(DynamicType::get())) { + for (auto & ret_arg : schema.returns) { + if (!ret_arg.type->isSubtypeOf(DynamicType::get())) { throw ErrorReport(loc) << "Python functions can currently only return Tensors"; } } std::vector outputs; - for(size_t i = 0; i < returns.size(); ++i) + for(size_t i = 0; i < schema.returns.size(); ++i) outputs.push_back(new_node->addOutput()); return packOutputs(*m.graph(), outputs); } @@ -155,13 +151,6 @@ struct VISIBILITY_HIDDEN PythonValue : public SugaredValue { } protected: - bool isBuiltinModule() { - // XXX: these can't be static, or they will be destructed after the Python interpreter - // exits and that generally sounds like a bad idea - py::object torch = py::module::import("torch"); - py::object functional = py::module::import("torch.nn.functional"); - return self.is(torch) || self.is(functional); - } py::object getattr(SourceRange loc, const std::string& name) { try { @@ -174,56 +163,48 @@ struct VISIBILITY_HIDDEN PythonValue : public SugaredValue { py::object self; }; -// by using torch.jit.Const, a user can mark a python value constant -// we then make that value immutable. -// once marked constant, we enable additional behavior such as -// 1. conversion via asValue to a constant Tensor -// 2. unrolling of for loops -struct VISIBILITY_HIDDEN ConstantPythonValue : public PythonValue { - using PythonValue::PythonValue; - virtual Value * asValue(SourceRange loc, Method & m) override { +struct VISIBILITY_HIDDEN PythonModuleValue : public PythonValue { + explicit PythonModuleValue(py::object mod) : PythonValue(mod) {} - return PythonValue::asValue(loc, m); + std::shared_ptr attr(SourceRange loc, Method & m, const std::string& field) override { + py::object member = getattr(loc, field); + return toSugaredValue(member, m, loc); } - virtual std::vector> asTuple(SourceRange loc, Method& m) override { - if(!py::isinstance(self)) - return PythonValue::asTuple(loc, m); + private: +}; + +struct VISIBILITY_HIDDEN BuiltinPythonModuleValue : public PythonModuleValue { + explicit BuiltinPythonModuleValue(py::object mod) : PythonModuleValue(mod) {} + std::shared_ptr attr(SourceRange loc, Method & m, const std::string& field) override { + // We support calling functions and using type/layout/device constants + // on the torch builtin modules + py::object member = getattr(loc, field); + if (py::isinstance(member)) { + return std::make_shared(field, at::nullopt); + } + return toSugaredValue(member, m, loc, /*is_constant =*/true); + } +}; +bool isBuiltinModule(py::object obj) { + // XXX: these can't be static, or they will be destructed after the Python interpreter + // exits and that generally sounds like a bad idea + py::object torch = py::module::import("torch"); + py::object functional = py::module::import("torch.nn.functional"); + return obj.is(torch) || obj.is(functional); +} + +struct VISIBILITY_HIDDEN ConstantPythonTupleValue : public PythonValue { + explicit ConstantPythonTupleValue(py::object tup) : PythonValue(tup) {} + std::vector> asTuple(SourceRange loc, Method& m) override { py::tuple tup = self; std::vector> result; - for(size_t i = 0; i < tup.size(); ++i) { - result.push_back(create(loc, m, tup[i])); + result.reserve(tup.size()); + for (size_t i = 0; i < tup.size(); ++i) { + result.push_back(toSugaredValue(tup[i], m, loc, true)); } return result; } - static std::shared_ptr create(SourceRange loc, Method& m, py::object self) { - // directly create SimpleValues when possible, because they are first-class - // and can be re-assigned. Otherwise, this would be invalid: - // f = python_constant - // while ... - // f = f + 1 - auto& g = *m.graph(); - if(py::isinstance(self)) { - return toSimple(insertConstant(g, py::cast(self), loc)); - } else if(py::isinstance(self)) { - return toSimple(insertConstant(g, py::cast(self), loc)); - } else if(py::isinstance(self)) { - return toSimple(insertConstant(g, py::cast(self), loc)); - } else if(THPDevice_Check(self.ptr())) { - auto device = (THPDevice*) self.ptr(); - std::vector v = {static_cast(device->device.type()), device->device.index()}; - return toSimple(insertConstant(g, std::move(v))); - } else if(THPLayout_Check(self.ptr())) { - auto layout = (THPLayout*) self.ptr(); - const auto v = static_cast(layout->layout); - return toSimple(insertConstant(g, v, loc)); - } else if(THPDtype_Check(self.ptr())) { - auto dtype = (THPDtype*)(self.ptr()); - const auto v = static_cast(dtype->scalar_type); - return toSimple(insertConstant(g, v, loc)); - } - return std::make_shared(self); - } }; std::shared_ptr PythonValue::attr(SourceRange loc, Method & m, const std::string& field) { @@ -231,32 +212,8 @@ std::shared_ptr PythonValue::attr(SourceRange loc, Method & m, con // make an exception for traversing modules because we want to be access // torch, torch.nn.functional, and the functions they expose. py::object member = getattr(loc, field); - if (isBuiltinModule()) { - if(py::isinstance(member)) { - return std::make_shared(field, at::nullopt); - } - //e.g. any tensor attribute objects such as torch.uint8 - if(THPDtype_Check(member.ptr()) || - THPLayout_Check(member.ptr()) || - THPDevice_Check(member.ptr())) { - return ConstantPythonValue::create(loc, m, member); - } - } - if (py::isinstance(self) && py::isinstance(member)) { - return std::make_shared(member); - } - throw ErrorReport(loc) << "unsupported attribute lookup on " << py::repr(self) << "."; -} -Resolver pythonResolver(ResolutionCallback rcb) { - return [=](const std::string& name) -> std::shared_ptr { - AutoGIL ag; - py::object obj = rcb(name); - if(obj.is(py::none())) { - return nullptr; - } - return std::make_shared(obj); - }; + return toSugaredValue(member, m, loc); } // defines how modules/methods behave inside the script subset. @@ -305,11 +262,10 @@ struct ModuleValue : public SugaredValue { // python method. If so return this as a python value. py::object py_module = py::cast(module); if(py::object attr = py::getattr(py_module, field.c_str(), py::none())) { - if(py::isinstance(attr) || - py::isinstance(attr, py::module::import("torch.nn").attr("Module"))) { - return std::make_shared(attr); - } else if(py_module.attr("_constants_set").contains(field.c_str())) { - return ConstantPythonValue::create(loc, m, attr); + if (py::isinstance(attr) || + py::isinstance(attr, py::module::import("torch.nn").attr("Module")) || + py_module.attr("_constants_set").contains(field.c_str())) { + return toSugaredValue(attr, m, loc, true); } else { throw ErrorReport(loc) << "attribute '" << field << "' of type '" << typeString(attr) << "' is not usable in a script method (did you forget to add it __constants__?)"; } @@ -329,20 +285,77 @@ struct ModuleValue : public SugaredValue { std::vector> result; for(py::handle module : py_module) { py::object obj = py::reinterpret_borrow(module); - if(py::isinstance(obj)) { - auto r = py::cast>(obj); - result.push_back(std::make_shared(r)); - } else { - result.push_back(ConstantPythonValue::create(loc, m, obj)); - } + result.push_back(toSugaredValue( + obj, + m, + loc, + /*is_constant =*/false, + /*is_submodule =*/true)); } return result; } -private: + private: std::shared_ptr module; }; +std::shared_ptr toSugaredValue( + py::object obj, + Method& m, + SourceRange loc, + bool is_constant, + bool is_submodule) { + // directly create SimpleValues when possible, because they are first-class + // and can be re-assigned. Otherwise, this would be invalid: + // f = python_constant + // while ... + // f = f + 1 + auto& g = *m.graph(); + if (is_constant) { + if (py::isinstance(obj)) { + return toSimple(insertConstant(g, py::cast(obj), loc)); + } else if (py::isinstance(obj)) { + return toSimple(insertConstant(g, py::cast(obj), loc)); + } else if (py::isinstance(obj)) { + return toSimple(insertConstant(g, py::cast(obj), loc)); + } else if (THPDevice_Check(obj.ptr())) { + auto device = (THPDevice*)obj.ptr(); + std::vector v = {static_cast(device->device.type()), + device->device.index()}; + return toSimple(insertConstant(g, std::move(v))); + } else if (THPLayout_Check(obj.ptr())) { + auto layout = (THPLayout*)obj.ptr(); + const auto v = static_cast(layout->layout); + return toSimple(insertConstant(g, v, loc)); + } else if (THPDtype_Check(obj.ptr())) { + auto dtype = (THPDtype*)(obj.ptr()); + const auto v = static_cast(dtype->scalar_type); + return toSimple(insertConstant(g, v, loc)); + } else if (py::isinstance(obj)) { + return std::make_shared(obj); + } + } + if (py::isinstance(obj)) { + auto mod = py::cast>(obj); + // In the case that this Python object is not a submodule, inline *ONLY + // PURE* ScriptModules. This allows us to call arbitrary @script functions + // within a scripting context while still enforcing that parameters from + // stateful submodules are properly accounted for. + if (!is_submodule && mod->get_parameters().size() != 0) { + throw ErrorReport() + << "Attempted to inline a Module with parameters. " + "Stateful modules to be inlined must be submodules of the callee."; + } + return std::make_shared(mod); + } else if (py::isinstance(obj)) { + if (isBuiltinModule(obj)) { + return std::make_shared(obj); + } else { + return std::make_shared(obj); + } + } + return std::make_shared(obj); +} py::object unpackVariableTensorList(std::vector outputs) { // if we don't tell pybind these are variables it chokes on the @@ -381,6 +394,19 @@ py::object runMethodFromPython(Method& m, py::args args) { return wrapStack(std::move(stack), m.graph()->outputs()); } +Resolver pythonResolver(ResolutionCallback rcb) { + return [=](const std::string& name, + Method& m, + const SourceRange& loc) -> std::shared_ptr { + AutoGIL ag; + py::object obj = rcb(name); + if (obj.is(py::none())) { + return nullptr; + } + return toSugaredValue(obj, m, loc); + }; +} + void initJitScriptBindings(PyObject* module) { auto m = py::handle(module).cast(); diff --git a/torch/csrc/jit/script/lexer.h b/torch/csrc/jit/script/lexer.h index 1694889d630d3..5543f104ea37d 100644 --- a/torch/csrc/jit/script/lexer.h +++ b/torch/csrc/jit/script/lexer.h @@ -34,6 +34,7 @@ namespace script { _(TK_EQUIVALENT, "equivalent", "<=>") \ _(TK_IDENT, "ident", "") \ _(TK_STRING, "string", "") \ + _(TK_STRINGLITERAL, "string_literal", "") \ _(TK_CONST, "const", "") \ _(TK_LIST, "list", "") \ _(TK_OPTION, "option", "") \ @@ -187,6 +188,42 @@ struct SharedParserData { *len = endptr - startptr; return *len > 0; } + + bool isCharCount(char c, const std::string& str, size_t start, int len) { + //count checks from [start, start + len) + return start + len <= str.size() && std::count(str.begin() + start, str.begin() + start + len, c) == len; + } + + // python conconcatenates all adjacent strings "a" "b" == "ab" + // strings can be enclosed with 1 or 3 single or double quotes + // if enclosed with 3 quotes newlines are valid + // as elsewhere, backslash and new line should be ignored + bool isString(const std::string& str, size_t start, size_t* len) { + char quote = str[start]; + if (quote != '\"' && quote != '\'') + return false; + int quote_len = isCharCount(quote, str, start, 3) ? 3 : 1; + + //end is now set past the opening quotation marks + size_t end = start + quote_len; + while(end < str.size() && !isCharCount(quote, str, end, quote_len)) { + if (str[end] == '\n' && quote_len != 3) { + return false; + } + //handle escaped characters. advances past escaped quotation marks, + //escaped newlines and escaped backslashes + if (str[end] == '\\') { + end++; + } + end++; + } + //set length equal to the complete string including quotations + *len = end - start + quote_len; + //if end finished without going past the last character of the string than + //there is a match + return end < str.size(); + } + bool isblank(int n) { return isspace(n) && n != '\n'; } @@ -244,6 +281,12 @@ struct SharedParserData { *kind = TK_NUMBER; return true; } + // check for string + if (isString(str, pos, len)) { + *kind = TK_STRINGLITERAL; + return true; + } + // check for either an ident or a token // ident tracks whether what we have scanned so far could be an identifier // matched indicates if we have found any match. diff --git a/torch/csrc/jit/script/parser.h b/torch/csrc/jit/script/parser.h index 0cd833dc15e48..189b72018220b 100644 --- a/torch/csrc/jit/script/parser.h +++ b/torch/csrc/jit/script/parser.h @@ -75,6 +75,9 @@ struct Parser { auto list = parseList('[', ',', ']', &Parser::parseExp); prefix = ListLiteral::create(list.range(), List(list)); } break; + case TK_STRINGLITERAL: { + prefix = parseStringLiteral(); + } break; default: { Ident name = parseIdent(); prefix = Var::create(name.range(), name); @@ -180,14 +183,67 @@ struct Parser { L.expect(end); return List::create(r, elements); } + Const parseConst() { auto range = L.cur().range; auto t = L.expect(TK_NUMBER); return Const::create(t.range, t.text()); } + + bool isCharCount(char c, const std::string& str, size_t start, int len) { + //count checks from [start, start + len) + return start + len <= str.size() && std::count(str.begin() + start, str.begin() + start + len, c) == len; + } + + std::string parseString(const SourceRange& range, const std::string &str) { + int quote_len = isCharCount(str[0], str, 0, 3) ? 3 : 1; + auto ret_str = str.substr(quote_len, str.size() - quote_len * 2); + size_t pos = ret_str.find('\\'); + while(pos != std::string::npos) { + //invariant: pos has to escape a character because it is a valid string + char c = ret_str[pos + 1]; + switch (ret_str[pos + 1]) { + case '\\': + case '\'': + case '\"': + case '\n': + break; + case 'a': + c = '\a'; + break; + case 'b': + c = '\b'; + break; + case 'f': + c = '\f'; + break; + case 'n': + c = '\n'; + break; + case 'v': + c = '\v'; + break; + default: + throw ErrorReport(range) << " octal and hex escaped sequences are not supported"; + } + ret_str.replace(pos, /* num to erase */ 2, /* num copies */ 1, c); + pos = ret_str.find('\\', pos + 1); + } + return ret_str; + } + + StringLiteral parseStringLiteral() { + auto range = L.cur().range; + std::stringstream ss; + while(L.cur().kind == TK_STRINGLITERAL) + ss << parseString(L.cur().range, L.next().text()); + return StringLiteral::create(range, ss.str()); + } + Expr parseAttributeValue() { return parseExp(); } + void parseOperatorArguments(TreeList& inputs, TreeList& attributes) { L.expect('('); if (L.cur().kind != ')') { diff --git a/torch/csrc/jit/script/python_tree_views.cpp b/torch/csrc/jit/script/python_tree_views.cpp index 7ece5e055a33d..b49673de25fc0 100644 --- a/torch/csrc/jit/script/python_tree_views.cpp +++ b/torch/csrc/jit/script/python_tree_views.cpp @@ -174,6 +174,10 @@ void initTreeViewBindings(PyObject *module) { .def(py::init([](const SourceRange& range, std::string value) { return Const::create(range, value); })); + py::class_(m, "StringLiteral") + .def(py::init([](const SourceRange& range, std::string value) { + return StringLiteral::create(range, value); + })); py::class_(m, "Apply") .def(py::init([](const Expr& expr, std::vector args, std::vector kwargs) { auto r = expr.range(); diff --git a/torch/csrc/jit/script/tree_views.h b/torch/csrc/jit/script/tree_views.h index 10ac01799c060..2ab648adf2e8d 100644 --- a/torch/csrc/jit/script/tree_views.h +++ b/torch/csrc/jit/script/tree_views.h @@ -246,6 +246,7 @@ struct Expr : public TreeView { case '/': case TK_NOT: case TK_CONST: + case TK_STRINGLITERAL: case TK_TRUE: case TK_FALSE: case TK_NONE: @@ -564,6 +565,18 @@ struct Const : public Expr { } }; +struct StringLiteral : public Expr { + explicit StringLiteral(const TreeRef& tree) : Expr(tree) { + tree_->matchNumSubtrees(TK_STRINGLITERAL, 1); + } + const std::string& text() const { + return subtree(0)->stringValue(); + } + static StringLiteral create(const SourceRange& range, const std::string& value) { + return StringLiteral(Compound::create(TK_STRINGLITERAL, range, {String::create(value)})); + } +}; + struct Apply : public Expr { explicit Apply(const TreeRef& tree) : Expr(tree) { tree_->match(TK_APPLY); diff --git a/torch/csrc/jit/type.cpp b/torch/csrc/jit/type.cpp index ddb4dfad0154a..7f246ce518a2f 100644 --- a/torch/csrc/jit/type.cpp +++ b/torch/csrc/jit/type.cpp @@ -40,6 +40,8 @@ std::ostream& operator<<(std::ostream & out, const Type & t) { out << *prim << "[]"; } else if(t.kind() == TypeKind::NoneType) { out << "None"; + } else if(t.kind() == TypeKind::StringType) { + out << "string"; } else { AT_ERROR("unknown type kind"); } @@ -66,6 +68,10 @@ NoneTypePtr NoneType::get() { static auto value = NoneType::create(); return value; } +StringTypePtr StringType::get() { + static auto value = StringType::create(); + return value; +} ListTypePtr ListType::ofTensors() { static auto value = ListType::create(DynamicType::get()); return value; diff --git a/torch/csrc/jit/type.h b/torch/csrc/jit/type.h index 713718e40681c..4febad6693845 100644 --- a/torch/csrc/jit/type.h +++ b/torch/csrc/jit/type.h @@ -21,6 +21,7 @@ _(NumberType) \ _(FloatType) \ _(IntType) \ _(NoneType) \ +_(StringType) \ enum class TypeKind { #define DEFINE_TYPE(T) T, @@ -382,6 +383,31 @@ struct TORCH_API IntType : public Type { : Type(TypeKind::IntType) {} }; +struct StringType; +using StringTypePtr = std::shared_ptr; +// This node represents a Python string value +struct TORCH_API StringType : public Type { + template + static StringTypePtr create( T&& ... all ) { + return StringTypePtr(new StringType( std::forward(all)... )); + } + bool operator==(const Type& rhs) const override { + return rhs.kind() == kind(); + } + std::string str() const override { + return "string"; + } + bool isSubtypeOf(const TypePtr rhs) const override { + return *this == *rhs; + } + static const TypeKind Kind = TypeKind::StringType; + // global singleton + static StringTypePtr get(); +private: + StringType() + : Type(TypeKind::StringType) {} +}; + struct NoneType; using NoneTypePtr = std::shared_ptr; // This node represents a Python int number value diff --git a/torch/functional.py b/torch/functional.py index 0133a01298185..116238bbea79a 100644 --- a/torch/functional.py +++ b/torch/functional.py @@ -209,12 +209,12 @@ def stft(input, n_fft, hop_length=None, win_length=None, window=None, expression: .. math:: - X[m, \omega] = \sum_{k = 0}^{\text{win_length}}% + X[m, \omega] = \sum_{k = 0}^{\text{win\_length}}% window[k]\ input[m \times hop_length + k]\ % - e^{- j \frac{2 \pi \cdot \omega k}{\text{win_length}}}, + e^{- j \frac{2 \pi \cdot \omega k}{\text{win\_length}}}, where :math:`m` is the index of the sliding window, and :math:`\omega` is - the frequency that :math:`0 \leq \omega < \text{n_fft}`. When + the frequency that :math:`0 \leq \omega < \text{n\_fft}`. When :attr:`onesided` is the default value ``True``, * :attr:`input` must be either a 1-D time sequenceor 2-D a batch of time @@ -229,25 +229,25 @@ def stft(input, n_fft, hop_length=None, win_length=None, window=None, * :attr:`window` can be a 1-D tensor of size :attr:`win_length`, e.g., from :meth:`torch.hann_window`. If :attr:`window` is ``None`` (default), it is treated as if having :math:`1` everywhere in the window. If - :math:`\text{win_length} < \text{n_fft}`, :attr:`window` will be padded on + :math:`\text{win\_length} < \text{n\_fft}`, :attr:`window` will be padded on both sides to length :attr:`n_fft` before being applied. * If :attr:`center` is ``True`` (default), :attr:`input` will be padded on both sides so that the :math:`t`-th frame is centered at time - :math:`t \times \text{hop_length}`. Otherwise, the :math:`t`-th frame - begins at time :math:`t \times \text{hop_length}`. + :math:`t \times \text{hop\_length}`. Otherwise, the :math:`t`-th frame + begins at time :math:`t \times \text{hop\_length}`. * :attr:`pad_mode` determines the padding method used on :attr:`input` when :attr:`center` is ``True``. See :meth:`torch.nn.functional.pad` for all available options. Default is ``"reflect"``. * If :attr:`onesided` is ``True`` (default), only values for :math:`\omega` - in :math:`\left[0, 1, 2, \dots, \left\lfloor \frac{\text{n_fft}}{2} \right\rfloor + 1\right]` + in :math:`\left[0, 1, 2, \dots, \left\lfloor \frac{\text{n\_fft}}{2} \right\rfloor + 1\right]` are returned because the real-to-complex Fourier transform satisfies the - conjugate symmetry, i.e., :math:`X[m, \omega] = X[m, \text{n_fft} - \omega]^*`. + conjugate symmetry, i.e., :math:`X[m, \omega] = X[m, \text{n\_fft} - \omega]^*`. * If :attr:`normalized` is ``True`` (default is ``False``), the function - returns the normalized STFT results, i.e., multiplied by :math:`(\text{frame_length})^{-0.5}`. + returns the normalized STFT results, i.e., multiplied by :math:`(\text{frame\_length})^{-0.5}`. Returns the real and the imaginary parts together as one tensor of size :math:`(* \times N \times T \times 2)`, where :math:`*` is the optional @@ -270,7 +270,7 @@ def stft(input, n_fft, hop_length=None, win_length=None, window=None, window (Tensor, optional): the optional window function. Default: ``None`` (treated as window of all :math:`1`s) center (bool, optional): whether to pad :attr:`input` on both sides so - that the :math:`t`-th frame is centered at time :math:`t \times \text{hop_length}`. + that the :math:`t`-th frame is centered at time :math:`t \times \text{hop\_length}`. Default: ``True`` pad_mode (string, optional): controls the padding method used when :attr:`center` is ``True``. Default: ``"reflect"`` diff --git a/torch/jit/frontend.py b/torch/jit/frontend.py index bc979d1514112..9cae1878139ef 100644 --- a/torch/jit/frontend.py +++ b/torch/jit/frontend.py @@ -444,6 +444,12 @@ def build_Num(ctx, expr): r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + len(value)) return Const(r, value) + @staticmethod + def build_Str(ctx, expr): + value = str(expr.s) + r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1) + return StringLiteral(r, value) + @staticmethod def build_Starred(ctx, expr): r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1) diff --git a/torch/nn/functional.py b/torch/nn/functional.py index 746c266452917..21c09c412af6f 100644 --- a/torch/nn/functional.py +++ b/torch/nn/functional.py @@ -765,7 +765,7 @@ def leaky_relu(input, negative_slope=0.01, inplace=False): leaky_relu(input, negative_slope=0.01, inplace=False) -> Tensor Applies element-wise, - :math:`\text{LeakyReLU}(x) = \max(0, x) + \text{negative_slope} * \min(0, x)` + :math:`\text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)` See :class:`~torch.nn.LeakyReLU` for more details. """ diff --git a/torch/nn/init.py b/torch/nn/init.py index ecc92606ecfbe..a5c8239d3dc96 100644 --- a/torch/nn/init.py +++ b/torch/nn/init.py @@ -17,7 +17,7 @@ def calculate_gain(nonlinearity, param=None): Sigmoid :math:`1` Tanh :math:`\frac{5}{3}` ReLU :math:`\sqrt{2}` - Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative_slope}^2}}` + Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}` ================= ==================================================== Args: @@ -203,7 +203,7 @@ def xavier_uniform_(tensor, gain=1): :math:`\mathcal{U}(-a, a)` where .. math:: - a = \text{gain} \times \sqrt{\frac{6}{\text{fan_in} + \text{fan_out}}} + a = \text{gain} \times \sqrt{\frac{6}{\text{fan\_in} + \text{fan\_out}}} Also known as Glorot initialization. @@ -230,7 +230,7 @@ def xavier_normal_(tensor, gain=1): :math:`\mathcal{N}(0, \text{std})` where .. math:: - \text{std} = \text{gain} \times \sqrt{\frac{2}{\text{fan_in} + \text{fan_out}}} + \text{std} = \text{gain} \times \sqrt{\frac{2}{\text{fan\_in} + \text{fan\_out}}} Also known as Glorot initialization. @@ -266,7 +266,7 @@ def kaiming_uniform_(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'): :math:`\mathcal{U}(-\text{bound}, \text{bound})` where .. math:: - \text{bound} = \sqrt{\frac{6}{(1 + a^2) \times \text{fan_in}}} + \text{bound} = \sqrt{\frac{6}{(1 + a^2) \times \text{fan\_in}}} Also known as He initialization. @@ -301,7 +301,7 @@ def kaiming_normal_(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'): :math:`\mathcal{N}(0, \text{std})` where .. math:: - \text{std} = \sqrt{\frac{2}{(1 + a^2) \times \text{fan_in}}} + \text{std} = \sqrt{\frac{2}{(1 + a^2) \times \text{fan\_in}}} Also known as He initialization. diff --git a/torch/nn/modules/activation.py b/torch/nn/modules/activation.py index 51cfab7940414..9c70d19ddd57b 100644 --- a/torch/nn/modules/activation.py +++ b/torch/nn/modules/activation.py @@ -82,17 +82,19 @@ def extra_repr(self): class RReLU(Module): - r"""Applies the randomized leaky rectified liner unit function element-wise - described in the paper + r"""Applies the randomized leaky rectified liner unit function, element-wise, + as described in the paper: + `Empirical Evaluation of Rectified Activations in Convolutional Network`_. The function is defined as: .. math:: - \text{RReLU}(x) = \begin{cases} + \text{RReLU}(x) = + \begin{cases} x & \text{if } x \geq 0 \\ ax & \text{ otherwise } - \end{cases}, + \end{cases} where :math:`a` is randomly sampled from uniform distribution :math:`\mathcal{U}(\text{lower}, \text{upper})`. @@ -195,7 +197,10 @@ def extra_repr(self): class ReLU6(Hardtanh): - r"""Applies the element-wise function :math:`\text{ReLU6}(x) = \min(\max(0,x), 6)` + r"""Applies the element-wise function: + + .. math:: + \text{ReLU6}(x) = \min(\max(0,x), 6) Args: inplace: can optionally do the operation in-place. Default: ``False`` @@ -223,7 +228,11 @@ def extra_repr(self): class Sigmoid(Module): - r"""Applies the element-wise function :math:`\text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}` + r"""Applies the element-wise function: + + .. math:: + \text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)} + Shape: - Input: :math:`(N, *)` where `*` means, any number of additional @@ -244,8 +253,10 @@ def forward(self, input): class Tanh(Module): - r"""Applies element-wise, - :math:`\text{Tanh}(x) = \tanh(x) = \frac{e^x - e^{-x}} {e^x + e^{-x}}` + r"""Applies the element-wise function: + + .. math:: + \text{Tanh}(x) = \tanh(x) = \frac{e^x - e^{-x}} {e^x + e^{-x}} Shape: - Input: :math:`(N, *)` where `*` means, any number of additional @@ -266,8 +277,10 @@ def forward(self, input): class ELU(Module): - r"""Applies element-wise, - :math:`\text{ELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1))` + r"""Applies the element-wise function: + + .. math:: + \text{ELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1)) Args: alpha: the :math:`\alpha` value for the ELU formulation. Default: 1.0 @@ -301,8 +314,10 @@ def extra_repr(self): class CELU(Module): - r"""Applies element-wise, - :math:`\text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))` + r"""Applies the element-wise function: + + .. math:: + \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1)) More details can be found in the paper `Continuously Differentiable Exponential Linear Units`_ . @@ -341,8 +356,11 @@ def extra_repr(self): class SELU(Module): - r"""Applies element-wise, - :math:`\text{SELU}(x) = \text{scale} * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))`, + r"""Applied element-wise, as: + + .. math:: + \text{SELU}(x) = \text{scale} * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1))) + with :math:`\alpha = 1.6732632423543772848170429916717` and :math:`\text{scale} = 1.0507009873554804934193349852946`. @@ -381,8 +399,8 @@ def extra_repr(self): class GLU(Module): r"""Applies the gated linear unit function - :math:`{GLU}(a, b)= a \otimes \sigma(b)` where `a` is the first half of - the input vector and `b` is the second half. + :math:`{GLU}(a, b)= a \otimes \sigma(b)` where :math:`a` is the first half + of the input vector and :math:`b` is the second half. Args: dim (int): the dimension on which to split the input. Default: -1 @@ -411,8 +429,7 @@ def extra_repr(self): class Hardshrink(Module): - r"""Applies the hard shrinkage function element-wise - Hardshrink is defined as: + r"""Applies the hard shrinkage function element-wise: .. math:: \text{HardShrink}(x) = @@ -451,14 +468,19 @@ def extra_repr(self): class LeakyReLU(Module): - r"""Applies element-wise, - :math:`\text{LeakyReLU}(x) = \max(0, x) + \text{negative_slope} * \min(0, x)` or + r"""Applies the element-wise function: + + .. math:: + \text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x) + + + or .. math:: \text{LeakyRELU}(x) = \begin{cases} x, & \text{ if } x \geq 0 \\ - \text{negative_slope} \times x, & \text{ otherwise } + \text{negative\_slope} \times x, & \text{ otherwise } \end{cases} Args: @@ -493,7 +515,9 @@ def extra_repr(self): class LogSigmoid(Module): - r"""Applies element-wise :math:`\text{LogSigmoid}(x) = \log\left(\frac{ 1 }{ 1 + \exp(-x)}\right)` + r"""Applies the element-wise function: + + .. math:`\text{LogSigmoid}(x) = \log\left(\frac{ 1 }{ 1 + \exp(-x)}\right)` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional @@ -514,7 +538,10 @@ def forward(self, input): class Softplus(Module): - r"""Applies element-wise :math:`\text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x))` + r"""Applies the element-wise function: + + .. math:: + \text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x)) SoftPlus is a smooth approximation to the ReLU function and can be used to constrain the output of a machine to always be positive. @@ -553,9 +580,7 @@ def extra_repr(self): class Softshrink(Module): - r"""Applies the soft shrinkage function elementwise - - SoftShrinkage function is defined as: + r"""Applies the soft shrinkage function elementwise: .. math:: \text{SoftShrinkage}(x) = @@ -594,8 +619,12 @@ def extra_repr(self): class PReLU(Module): - r"""Applies element-wise the function - :math:`\text{PReLU}(x) = \max(0,x) + a * \min(0,x)` or + r"""Applies the element-wise function: + + .. math:: + \text{PReLU}(x) = \max(0,x) + a * \min(0,x) + + or .. math:: \text{PReLU}(x) = @@ -643,7 +672,10 @@ def extra_repr(self): class Softsign(Module): - r"""Applies element-wise, the function :math:`\text{SoftSign}(x) = \frac{x}{ 1 + |x|}` + r"""Applies the element-wise function: + + .. math:: + \text{SoftSign}(x) = \frac{x}{ 1 + |x|} Shape: - Input: :math:`(N, *)` where `*` means, any number of additional @@ -664,7 +696,10 @@ def forward(self, input): class Tanhshrink(Module): - r"""Applies element-wise, :math:`\text{Tanhshrink}(x) = x - \text{Tanh}(x)` + r"""Applies the element-wise function: + + .. math:: + \text{Tanhshrink}(x) = x - \text{Tanh}(x) Shape: - Input: :math:`(N, *)` where `*` means, any number of additional @@ -689,7 +724,8 @@ class Softmin(Module): rescaling them so that the elements of the n-dimensional output Tensor lie in the range `(0, 1)` and sum to 1 - :math:`\text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)}` + .. math:: + \text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)} Shape: - Input: any shape @@ -723,8 +759,10 @@ class Softmax(Module): rescaling them so that the elements of the n-dimensional output Tensor lie in the range (0,1) and sum to 1 - Softmax is defined as - :math:`\text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}` + Softmax is defined as: + + .. math:: + \text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)} Shape: - Input: any shape @@ -791,10 +829,11 @@ def forward(self, input): class LogSoftmax(Module): - r"""Applies the `Log(Softmax(x))` function to an n-dimensional input Tensor. - The LogSoftmax formulation can be simplified as + r"""Applies the :math:`\log(\text{Softmax}(x))` function to an n-dimensional + input Tensor. The LogSoftmax formulation can be simplified as: - :math:`\text{LogSoftmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right)` + .. math:: + \text{LogSoftmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right) Shape: - Input: any shape diff --git a/torch/nn/modules/conv.py b/torch/nn/modules/conv.py index 5af7d29231acc..03cf297e3c0aa 100644 --- a/torch/nn/modules/conv.py +++ b/torch/nn/modules/conv.py @@ -67,15 +67,13 @@ class Conv1d(_ConvNd): planes. In the simplest case, the output value of the layer with input size - :math:`(N, C_{in}, L)` and output :math:`(N, C_{out}, L_{out})` can be + :math:`(N, C_{\text{in}}, L)` and output :math:`(N, C_{\text{out}}, L_{\text{out}})` can be precisely described as: .. math:: - - \begin{equation*} - \text{out}(N_i, C_{out_j}) = \text{bias}(C_{out_j}) + - \sum_{k = 0}^{C_{in} - 1} \text{weight}(C_{out_j}, k) \star \text{input}(N_i, k) - \end{equation*}, + \op{out}(N_i, C_{\text{out}_j}) = \op{bias}(C_{\text{out}_j}) + + \sum_{k = 0}^{C_{in} - 1} \op{weight}(C_{\text{out}_j}, k) + \star \op{input}(N_i, k) where :math:`\star` is the valid `cross-correlation`_ operator, :math:`N` is a batch size, :math:`C` denotes a number of channels, @@ -101,8 +99,9 @@ class Conv1d(_ConvNd): and producing half the output channels, and both subsequently concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with - its own set of filters (of size - :math:`\left\lfloor \frac{\text{out_channels}}{\text{in_channels}} \right\rfloor`). + its own set of filters, + of size + :math:`\left\lfloor\frac{\text{out\_channels}}{\text{in\_channels}}\right\rfloor` .. note:: @@ -119,7 +118,7 @@ class Conv1d(_ConvNd): In other words, for an input of size :math:`(N, C_{in}, L_{in})`, if you want a depthwise convolution with a depthwise multiplier `K`, then you use the constructor arguments - :math:`(\text{in_channels}=C_{in}, \text{out_channels}=C_{in} * K, ..., \text{groups}=C_{in})` + :math:`(\text{in\_channels}=C_{in}, \text{out\_channels}=C_{in} * K, ..., \text{groups}=C_{in})` Args: in_channels (int): Number of channels in the input image @@ -140,17 +139,17 @@ class Conv1d(_ConvNd): .. math:: L_{out} = \left\lfloor\frac{L_{in} + 2 \times \text{padding} - \text{dilation} - \times (\text{kernel_size} - 1) - 1}{\text{stride}} + 1\right\rfloor + \times (\text{kernel\_size} - 1) - 1}{\text{stride}} + 1\right\rfloor Attributes: weight (Tensor): the learnable weights of the module of shape (out_channels, in_channels, kernel_size). The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \text{kernel_size}}` + :math:`k = \frac{1}{\text{in\_channels} * \text{kernel\_size}}` bias (Tensor): the learnable bias of the module of shape (out_channels). If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \text{kernel_size}}` + :math:`k = \frac{1}{\text{in\_channels} * \text{kernel\_size}}` Examples:: @@ -185,15 +184,13 @@ class Conv2d(_ConvNd): planes. In the simplest case, the output value of the layer with input size - :math:`(N, C_{in}, H, W)` and output :math:`(N, C_{out}, H_{out}, W_{out})` + :math:`(N, C_{\text{in}}, H, W)` and output :math:`(N, C_{\text{out}}, H_{\text{out}}, W_{\text{out}})` can be precisely described as: .. math:: + \op{out}(N_i, C_{\text{out}_j}) = \op{bias}(C_{\text{out}_j}) + + \sum_{k = 0}^{C_{\text{in}} - 1} \op{weight}(C_{\text{out}_j}, k) \star \op{input}(N_i, k) - \begin{equation*} - \text{out}(N_i, C_{out_j}) = \text{bias}(C_{out_j}) + - \sum_{k = 0}^{C_{in} - 1} \text{weight}(C_{out_j}, k) \star \text{input}(N_i, k) - \end{equation*}, where :math:`\star` is the valid 2D `cross-correlation`_ operator, :math:`N` is a batch size, :math:`C` denotes a number of channels, @@ -220,8 +217,8 @@ class Conv2d(_ConvNd): and producing half the output channels, and both subsequently concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with - its own set of filters (of size - :math:`\left\lfloor\frac{\text{out_channels}}{\text{in_channels}}\right\rfloor`). + its own set of filters, of size: + :math:`\left\lfloor\frac{\text{out\_channels}}{\text{in\_channels}}\right\rfloor`. The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`dilation` can either be: @@ -244,7 +241,7 @@ class Conv2d(_ConvNd): In other words, for an input of size :math:`(N, C_{in}, H_{in}, W_{in})`, if you want a depthwise convolution with a depthwise multiplier `K`, then you use the constructor arguments - :math:`(\text{in_channels}=C_{in}, \text{out_channels}=C_{in} * K, ..., \text{groups}=C_{in})` + :math:`(in\_channels=C_{in}, out\_channels=C_{in} * K, ..., groups=C_{in})` Args: in_channels (int): Number of channels in the input image @@ -262,21 +259,21 @@ class Conv2d(_ConvNd): .. math:: H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[0] - \text{dilation}[0] - \times (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + \times (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[1] - \text{dilation}[1] - \times (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + \times (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor Attributes: weight (Tensor): the learnable weights of the module of shape (out_channels, in_channels, kernel_size[0], kernel_size[1]). The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{1}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{1}\text{kernel\_size}[i]}` bias (Tensor): the learnable bias of the module of shape (out_channels). If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{1}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{1}\text{kernel\_size}[i]}` Examples:: @@ -319,11 +316,8 @@ class Conv3d(_ConvNd): and output :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})` can be precisely described as: .. math:: - - \begin{equation*} - \text{out}(N_i, C_{out_j}) = \text{bias}(C_{out_j}) + - \sum_{k = 0}^{C_{in} - 1} \text{weight}(C_{out_j}, k) \star \text{input}(N_i, k) - \end{equation*}, + out(N_i, C_{out_j}) = bias(C_{out_j}) + + \sum_{k = 0}^{C_{in} - 1} weight(C_{out_j}, k) \star input(N_i, k) where :math:`\star` is the valid 3D `cross-correlation`_ operator @@ -345,8 +339,8 @@ class Conv3d(_ConvNd): and producing half the output channels, and both subsequently concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with - its own set of filters (of size - :math:`\left\lfloor\frac{\text{out_channels}}{\text{in_channels}}\right\rfloor`). + its own set of filters, of size + :math:`\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor`. The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`dilation` can either be: @@ -369,7 +363,7 @@ class Conv3d(_ConvNd): In other words, for an input of size :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`, if you want a depthwise convolution with a depthwise multiplier `K`, then you use the constructor arguments - :math:`(\text{in_channels}=C_{in}, \text{out_channels}=C_{in} * K, ..., \text{groups}=C_{in})` + :math:`(in\_channels=C_{in}, out\_channels=C_{in} * K, ..., groups=C_{in})` Args: in_channels (int): Number of channels in the input image @@ -387,24 +381,24 @@ class Conv3d(_ConvNd): .. math:: D_{out} = \left\lfloor\frac{D_{in} + 2 \times \text{padding}[0] - \text{dilation}[0] - \times (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + \times (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[1] - \text{dilation}[1] - \times (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + \times (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[2] - \text{dilation}[2] - \times (\text{kernel_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor + \times (\text{kernel\_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor Attributes: weight (Tensor): the learnable weights of the module of shape (out_channels, in_channels, kernel_size[0], kernel_size[1], kernel_size[2]) The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{2}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{2}\text{kernel\_size}[i]}` bias (Tensor): the learnable bias of the module of shape (out_channels). If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{2}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{2}\text{kernel\_size}[i]}` Examples:: @@ -509,7 +503,7 @@ class ConvTranspose1d(_ConvTransposeMixin, _ConvNd): concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with its own set of filters (of size - :math:`\left\lfloor\frac{\text{out_channels}}{\text{in_channels}}\right\rfloor`). + :math:`\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor`). .. note:: @@ -549,18 +543,18 @@ class ConvTranspose1d(_ConvTransposeMixin, _ConvNd): .. math:: L_{out} = (L_{in} - 1) \times \text{stride} - 2 \times \text{padding} - + \text{kernel_size} + \text{output_padding} + + \text{kernel\_size} + \text{output\_padding} Attributes: weight (Tensor): the learnable weights of the module of shape (in_channels, out_channels, kernel_size[0], kernel_size[1]). The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \text{kernel_size}}` + :math:`k = \frac{1}{\text{in\_channels} * \text{kernel\_size}}` bias (Tensor): the learnable bias of the module of shape (out_channels). If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \text{kernel_size}}` + :math:`k = \frac{1}{\text{in\_channels} * \text{kernel\_size}}` """ def __init__(self, in_channels, out_channels, kernel_size, stride=1, @@ -612,7 +606,7 @@ class ConvTranspose2d(_ConvTransposeMixin, _ConvNd): concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with its own set of filters (of size - :math:`\left\lfloor\frac{\text{out_channels}}{\text{in_channels}}\right\rfloor`). + :math:`\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor`). The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`output_padding` can either be: @@ -659,21 +653,21 @@ class ConvTranspose2d(_ConvTransposeMixin, _ConvNd): .. math:: H_{out} = (H_{in} - 1) \times \text{stride}[0] - 2 \times \text{padding}[0] - + \text{kernel_size}[0] + \text{output_padding}[0] + + \text{kernel\_size}[0] + \text{output\_padding}[0] W_{out} = (W_{in} - 1) \times \text{stride}[1] - 2 \times \text{padding}[1] - + \text{kernel_size}[1] + \text{output_padding}[1] + + \text{kernel\_size}[1] + \text{output\_padding}[1] Attributes: weight (Tensor): the learnable weights of the module of shape (in_channels, out_channels, kernel_size[0], kernel_size[1]) The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{1}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{1}\text{kernel\_size}[i]}` bias (Tensor): the learnable bias of the module of shape (out_channels) If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{1}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{1}\text{kernel\_size}[i]}` Examples:: @@ -752,7 +746,7 @@ class ConvTranspose3d(_ConvTransposeMixin, _ConvNd): concatenated. * At groups= :attr:`in_channels`, each input channel is convolved with its own set of filters (of size - :math:`\left\lfloor\frac{\text{out_channels}}{\text{in_channels}}\right\rfloor`). + :math:`\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor`). The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`output_padding` can either be: @@ -799,24 +793,24 @@ class ConvTranspose3d(_ConvTransposeMixin, _ConvNd): .. math:: D_{out} = (D_{in} - 1) \times \text{stride}[0] - 2 \times \text{padding}[0] - + \text{kernel_size}[0] + \text{output_padding}[0] + + \text{kernel\_size}[0] + \text{output\_padding}[0] H_{out} = (H_{in} - 1) \times \text{stride}[1] - 2 \times \text{padding}[1] - + \text{kernel_size}[1] + \text{output_padding}[1] + + \text{kernel\_size}[1] + \text{output\_padding}[1] W_{out} = (W_{in} - 1) \times \text{stride}[2] - 2 \times \text{padding}[2] - + \text{kernel_size}[2] + \text{output_padding}[2] + + \text{kernel\_size}[2] + \text{output\_padding}[2] Attributes: weight (Tensor): the learnable weights of the module of shape (in_channels, out_channels, kernel_size[0], kernel_size[1], kernel_size[2]) The values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{2}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{2}\text{kernel\_size}[i]}` bias (Tensor): the learnable bias of the module of shape (out_channels) If :attr:`bias` is ``True``, then the values of these weights are sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_channels} * \prod_{i=0}^{2}\text{kernel_size[i]}}` + :math:`k = \frac{1}{\text{in\_channels} * \prod_{i=0}^{2}\text{kernel\_size}[i]}` Examples:: diff --git a/torch/nn/modules/fold.py b/torch/nn/modules/fold.py index 580528e34a5b5..4465b9b10f0e7 100644 --- a/torch/nn/modules/fold.py +++ b/torch/nn/modules/fold.py @@ -8,19 +8,19 @@ class Fold(Module): tensor. Consider a batched :attr:`input` tensor containing sliding local blocks, - e.g., patches of images, of shape :math:`(N, C \times \prod(\text{kernel_size}), L)`, - where :math:`N` is batch dimension, :math:`C \times \prod(\text{kernel_size})` - is the number of values with in a block (a block has :math:`\prod(\text{kernel_size})` + e.g., patches of images, of shape :math:`(N, C \times \prod(\text{kernel\_size}), L)`, + where :math:`N` is batch dimension, :math:`C \times \prod(\text{kernel\_size})` + is the number of values with in a block (a block has :math:`\prod(\text{kernel\_size})` spatial locations each containing a :math:`C`-channeled vector), and :math:`L` is the total number of blocks. (This is exacly the same specification as the output shape of :class:`~torch.nn.Unfold`.) This operation combines these local blocks into the large :attr:`output` tensor - of shape :math:`(N, C, \text{output_size}[0], \text{output_size}[1], \dots)`. + of shape :math:`(N, C, \text{output\_size}[0], \text{output\_size}[1], \dots)`. Similar to :class:`~torch.nn.Unfold`, the arguments must satisfy .. math:: - L = \prod_d \left\lfloor\frac{\text{output_size}[d] + 2 \times \text{padding}[d] \ - - \text{dilation}[d] \times (\text{kernel_size}[d] - 1) - 1}{\text{stride}[d]} + 1\right\rfloor, + L = \prod_d \left\lfloor\frac{\text{output\_size}[d] + 2 \times \text{padding}[d] \ + - \text{dilation}[d] \times (\text{kernel\_size}[d] - 1) - 1}{\text{stride}[d]} + 1\right\rfloor, where :math:`d` is over all spatial dimensions. @@ -64,8 +64,8 @@ class Fold(Module): supported. Shape: - - Input: :math:`(N, C \times \prod(\text{kernel_size}), L)` - - Output: :math:`(N, C, \text{output_size}[0], \text{output_size}[1], \dots)` as described above + - Input: :math:`(N, C \times \prod(\text{kernel\_size}), L)` + - Output: :math:`(N, C, \text{output\_size}[0], \text{output\_size}[1], \dots)` as described above Examples:: @@ -106,17 +106,17 @@ class Unfold(Module): and :math:`*` represent arbitrary spatial dimensions. This operation flattens each sliding :attr:`kernel_size`-sized block within the spatial dimensions of :attr:`input` into a column (i.e., last dimension) of a 3-D :attr:`output` - tensor of shape :math:`(N, C \times \prod(\text{kernel_size}), L)`, where - :math:`C \times \prod(\text{kernel_size})` is the total number of values - with in each block (a block has :math:`\prod(\text{kernel_size})` spatial + tensor of shape :math:`(N, C \times \prod(\text{kernel\_size}), L)`, where + :math:`C \times \prod(\text{kernel\_size})` is the total number of values + with in each block (a block has :math:`\prod(\text{kernel\_size})` spatial locations each containing a :math:`C`-channeled vector), and :math:`L` is the total number of such blocks: .. math:: - L = \prod_d \left\lfloor\frac{\text{input_spatial_size}[d] + 2 \times \text{padding}[d] \ - - \text{dilation}[d] \times (\text{kernel_size}[d] - 1) - 1}{\text{stride}[d]} + 1\right\rfloor, + L = \prod_d \left\lfloor\frac{\text{input\_spatial\_size}[d] + 2 \times \text{padding}[d] \ + - \text{dilation}[d] \times (\text{kernel\_size}[d] - 1) - 1}{\text{stride}[d]} + 1\right\rfloor, - where :math:`\text{input_spatial_size}` is formed by the spatial dimensions + where :math:`\text{input\_spatial\_size}` is formed by the spatial dimensions of :attr:`input` (:math:`*` above), and :math:`d` is over all spatial dimensions. @@ -158,7 +158,7 @@ class Unfold(Module): Shape: - Input: :math:`(N, C, *)` - - Output: :math:`(N, C \times \prod(\text{kernel_size}), L)` as described above + - Output: :math:`(N, C \times \prod(\text{kernel\_size}), L)` as described above Examples:: diff --git a/torch/nn/modules/linear.py b/torch/nn/modules/linear.py index 429340aabc72e..573c6b20cbf03 100644 --- a/torch/nn/modules/linear.py +++ b/torch/nn/modules/linear.py @@ -26,11 +26,11 @@ class Linear(Module): weight: the learnable weights of the module of shape `(out_features x in_features)`. The values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_features}}` - bias: the learnable bias of the module of shape `(out_features)`. + :math:`k = \frac{1}{\text{in\_features}}` + bias: the learnable bias of the module of shape :math:`(out_features)`. If :attr:`bias` is ``True``, the values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in_features}}` + :math:`k = \frac{1}{\text{in\_features}}` Examples:: @@ -79,21 +79,21 @@ class Bilinear(Module): Default: ``True`` Shape: - - Input: :math:`(N, *, \text{in1_features})`, :math:`(N, *, \text{in2_features})` + - Input: :math:`(N, *, \text{in1\_features})`, :math:`(N, *, \text{in2\_features})` where :math:`*` means any number of additional dimensions. All but the last dimension of the inputs should be the same. - - Output: :math:`(N, *, \text{out_features})` where all but the last dimension + - Output: :math:`(N, *, \text{out\_features})` where all but the last dimension are the same shape as the input. Attributes: weight: the learnable weights of the module of shape `(out_features x in1_features x in2_features)`. The values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in1_features}}` + :math:`k = \frac{1}{\text{in1\_features}}` bias: the learnable bias of the module of shape `(out_features)` If :attr:`bias` is ``True``, the values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where - :math:`k = \frac{1}{\text{in1_features}}` + :math:`k = \frac{1}{\text{in1\_features}}` Examples:: diff --git a/torch/nn/modules/loss.py b/torch/nn/modules/loss.py index ec7d60d812515..d65e7a0b4c6d3 100644 --- a/torch/nn/modules/loss.py +++ b/torch/nn/modules/loss.py @@ -36,12 +36,13 @@ class L1Loss(_Loss): where :math:`N` is the batch size. If reduce is ``True``, then: .. math:: - \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \ell(x, y) = + \begin{cases} + \operatorname{mean}(L), & \text{if size\_average} = \text{True;}\\ + \operatorname{sum}(L), & \text{if size\_average} = \text{False.} \end{cases} - `x` and `y` arbitrary shapes with a total of `n` elements each. + `x` and `y` are tensors of arbitrary shapes with a total of `n` elements each. The sum operation still operates over all the elements, and divides by `n`. @@ -113,7 +114,7 @@ class NLLLoss(_WeightedLoss): .. math:: \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad l_n = - w_{y_n} x_{n,y_n}, \quad - w_{c} = \text{weight}[c] \cdot \mathbb{1}\{c \not= \text{ignore_index}\}, + w_{c} = \text{weight}[c] \cdot \mathbb{1}\{c \not= \text{ignore\_index}\}, where :math:`N` is the batch size. If :attr:`reduce` is ``True`` (default), then @@ -121,9 +122,9 @@ class NLLLoss(_WeightedLoss): .. math:: \ell(x, y) = \begin{cases} \sum_{n=1}^N \frac{1}{\sum_{n=1}^N w_{y_n}} l_n, & \text{if}\; - \text{size_average} = \text{True},\\ + \text{size\_average} = \text{True},\\ \sum_{n=1}^N l_n, & \text{if}\; - \text{size_average} = \text{False}. + \text{size\_average} = \text{False}. \end{cases} Can also be used for higher dimension inputs, such as 2D images, by providing @@ -290,15 +291,15 @@ class KLDivLoss(_Loss): .. math:: l(x,y) = L := \{ l_1,\dots,l_N \}, \quad - l_n = y_n \cdot \left( \log y_n - x_n \right), + l_n = y_n \cdot \left( \log y_n - x_n \right) where the index :math:`N` spans all dimensions of ``input`` and :math:`L` has the same shape as ``input``. If :attr:`reduce` is ``True`` (the default), then: .. math:: \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \operatorname{mean}(L), & \text{if}\; \text{size\_average} = \text{True},\\ + \operatorname{sum}(L), & \text{if}\; \text{size\_average} = \text{False}. \end{cases} By default, the losses are averaged for each minibatch over observations @@ -371,9 +372,10 @@ class MSELoss(_Loss): where :math:`N` is the batch size. If reduce is ``True``, then: .. math:: - \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \ell(x, y) = + \begin{cases} + \operatorname{mean}(L), & \text{if}\; \text{size\_average} = \text{True},\\ + \operatorname{sum}(L), & \text{if}\; \text{size\_average} = \text{False}. \end{cases} The sum operation still operates over all the elements, and divides by `n`. @@ -435,8 +437,8 @@ class BCELoss(_WeightedLoss): .. math:: \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \operatorname{mean}(L), & \text{if}\; \text{size\_average} = \text{True},\\ + \operatorname{sum}(L), & \text{if}\; \text{size\_average} = \text{False}. \end{cases} This is used for measuring the error of a reconstruction in for example @@ -503,8 +505,8 @@ class BCEWithLogitsLoss(_Loss): .. math:: \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \operatorname{mean}(L), & \text{if size\_average} = \text{True},\\ + \operatorname{sum}(L), & \text{if size\_average} = \text{False}. \end{cases} This is used for measuring the error of a reconstruction in for example @@ -592,8 +594,8 @@ class HingeEmbeddingLoss(_Loss): .. math:: \ell(x, y) = \begin{cases} - \operatorname{mean}(L), & \text{if}\; \text{size_average} = \text{True},\\ - \operatorname{sum}(L), & \text{if}\; \text{size_average} = \text{False}. + \operatorname{mean}(L), & \text{if size\_average} = \text{True},\\ + \operatorname{sum}(L), & \text{if size\_average} = \text{False}. \end{cases} where :math:`L = \{l_1,\dots,l_N\}^\top`. @@ -639,8 +641,10 @@ class MultiLabelMarginLoss(_Loss): .. math:: \text{loss}(x, y) = \sum_{ij}\frac{\max(0, 1 - (x[y[j]] - x[i]))}{\text{x.size}(0)} - where `i == 0` to `x.size(0)`, `j == 0` to `y.size(0)`, - :math:`y[j] \geq 0`, and :math:`i \neq y[j]` for all `i` and `j`. + where :math:`i == 0` to :math:`x.size(0)`, \ + :math:`j == 0` to :math:`y.size(0)`, \ + :math:`y[j] \geq 0`, \ + and :math:`i \neq y[j]` for all :math:`i` and :math:`j`. `y` and `x` must have the same size. @@ -1072,7 +1076,11 @@ class TripletMarginLoss(_Loss): .. math:: L(a, p, n) = \max \{d(a_i, p_i) - d(a_i, n_i) + {\rm margin}, 0\} - where :math:`d(x_i, y_i) = \left\lVert {\bf x}_i - {\bf y}_i \right\rVert_p`. + + where + + .. math:: + d(x_i, y_i) = \left\lVert {\bf x}_i - {\bf y}_i \right\rVert_p Args: margin (float, optional): Default: `1`. diff --git a/torch/nn/modules/normalization.py b/torch/nn/modules/normalization.py index a40cb314a6b8f..b3bf05f9d5307 100644 --- a/torch/nn/modules/normalization.py +++ b/torch/nn/modules/normalization.py @@ -95,8 +95,8 @@ class LayerNorm(Module): of size .. math:: - [* \times \text{normalized_shape}[0] \times \text{normalized_shape}[1] - \times \ldots \times \text{normalized_shape}[-1]] + [* \times \text{normalized\_shape}[0] \times \text{normalized\_shape}[1] + \times \ldots \times \text{normalized\_shape}[-1]] If a single integer is used, it is treated as a singleton list, and this module will normalize over the last dimension which is expected to be of that specific size. diff --git a/torch/nn/modules/padding.py b/torch/nn/modules/padding.py index 2ad9a5624a6a3..6358e8e0dae20 100644 --- a/torch/nn/modules/padding.py +++ b/torch/nn/modules/padding.py @@ -26,39 +26,39 @@ class ConstantPad1d(_ConstantPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in both boundaries. If a 2-`tuple`, uses (`paddingLeft`, `paddingRight`) + padding in both boundaries. If a 2-`tuple`, uses + (:math:`\text{padding\_left}`, :math:`\text{padding\_right}`) Shape: - Input: :math:`(N, C, W_{in})` - Output: :math:`(N, C, W_{out})` where - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ConstantPad1d(2, 3.5) >>> input = torch.randn(1, 2, 4) >>> input - - (0 ,.,.) = - 0.1875 0.5046 -1.0074 2.0005 - -0.3540 -1.8645 1.1530 0.0632 - [torch.FloatTensor of size (1,2,4)] - + tensor([[[-1.0491, -0.7152, -0.0749, 0.8530], + [-1.3287, 1.8966, 0.1466, -0.2771]]]) >>> m(input) - - (0 ,.,.) = - 3.5000 3.5000 0.1875 0.5046 -1.0074 2.0005 3.5000 3.5000 - 3.5000 3.5000 -0.3540 -1.8645 1.1530 0.0632 3.5000 3.5000 - [torch.FloatTensor of size (1,2,8)] - - >>> # using different paddings + tensor([[[ 3.5000, 3.5000, -1.0491, -0.7152, -0.0749, 0.8530, 3.5000, + 3.5000], + [ 3.5000, 3.5000, -1.3287, 1.8966, 0.1466, -0.2771, 3.5000, + 3.5000]]]) + >>> m = nn.ConstantPad1d(2, 3.5) + >>> input = torch.randn(1, 2, 3) + >>> input + tensor([[[ 1.6616, 1.4523, -1.1255], + [-3.6372, 0.1182, -1.8652]]]) + >>> m(input) + tensor([[[ 3.5000, 3.5000, 1.6616, 1.4523, -1.1255, 3.5000, 3.5000], + [ 3.5000, 3.5000, -3.6372, 0.1182, -1.8652, 3.5000, 3.5000]]]) + >>> # using different paddings for different sides >>> m = nn.ConstantPad1d((3, 1), 3.5) >>> m(input) - - (0 ,.,.) = - 3.5000 3.5000 3.5000 0.1875 0.5046 -1.0074 2.0005 3.5000 - 3.5000 3.5000 3.5000 -0.3540 -1.8645 1.1530 0.0632 3.5000 - [torch.FloatTensor of size (1,2,8)] + tensor([[[ 3.5000, 3.5000, 3.5000, 1.6616, 1.4523, -1.1255, 3.5000], + [ 3.5000, 3.5000, 3.5000, -3.6372, 0.1182, -1.8652, 3.5000]]]) """ @@ -74,48 +74,44 @@ class ConstantPad2d(_ConstantPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 4-`tuple`, uses (`paddingLeft`, `paddingRight`, - `paddingTop`, `paddingBottom`) + padding in all boundaries. If a 4-`tuple`, uses (:math:`\text{padding\_left}`, + :math:`\text{padding\_right}`, :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`) Shape: - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ConstantPad2d(2, 3.5) >>> input = torch.randn(1, 2, 2) >>> input - - (0 ,.,.) = - -0.2295 -0.9774 - -0.3335 -1.4178 - [torch.FloatTensor of size (1,2,2)] - + tensor([[[ 1.6585, 0.4320], + [-0.8701, -0.4649]]]) >>> m(input) - - (0 ,.,.) = - 3.5000 3.5000 3.5000 3.5000 3.5000 3.5000 - 3.5000 3.5000 3.5000 3.5000 3.5000 3.5000 - 3.5000 3.5000 -0.2295 -0.9774 3.5000 3.5000 - 3.5000 3.5000 -0.3335 -1.4178 3.5000 3.5000 - 3.5000 3.5000 3.5000 3.5000 3.5000 3.5000 - 3.5000 3.5000 3.5000 3.5000 3.5000 3.5000 - [torch.FloatTensor of size (1,6,6)] - - >>> # using different paddings + tensor([[[ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 1.6585, 0.4320, 3.5000, 3.5000], + [ 3.5000, 3.5000, -0.8701, -0.4649, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000]]]) + >>> m(input) + tensor([[[ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 1.6585, 0.4320, 3.5000, 3.5000], + [ 3.5000, 3.5000, -0.8701, -0.4649, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000, 3.5000]]]) + >>> # using different paddings for different sides >>> m = nn.ConstantPad2d((3, 0, 2, 1), 3.5) >>> m(input) - - (0 ,.,.) = - 3.5000 3.5000 3.5000 3.5000 3.5000 - 3.5000 3.5000 3.5000 3.5000 3.5000 - 3.5000 3.5000 3.5000 -0.2295 -0.9774 - 3.5000 3.5000 3.5000 -0.3335 -1.4178 - 3.5000 3.5000 3.5000 3.5000 3.5000 - [torch.FloatTensor of size (1,5,5)] + tensor([[[ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000], + [ 3.5000, 3.5000, 3.5000, 1.6585, 0.4320], + [ 3.5000, 3.5000, 3.5000, -0.8701, -0.4649], + [ 3.5000, 3.5000, 3.5000, 3.5000, 3.5000]]]) """ @@ -132,21 +128,23 @@ class ConstantPad3d(_ConstantPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same padding in all boundaries. If a 6-`tuple`, uses - (`paddingLeft`, `paddingRight`, `paddingTop`, `paddingBottom`, `paddingFront`, `paddingBack`) + (:math:`\text{padding\_left}`, :math:`\text{padding\_right}`, + :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`, + :math:`\text{padding\_front}`, :math:`\text{padding\_back}`) Shape: - Input: :math:`(N, C, D_{in}, H_{in}, W_{in})` - Output: :math:`(N, C, D_{out}, H_{out}, W_{out})` where - :math:`D_{out} = D_{in} + \textit{paddingFront} + \textit{paddingBack}` - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`D_{out} = D_{in} + \text{padding\_front} + \text{padding\_back}` + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ConstantPad3d(3, 3.5) >>> input = torch.randn(16, 3, 10, 20, 30) >>> output = m(input) - >>> # using different paddings + >>> # using different paddings for different sides >>> m = nn.ConstantPad3d((3, 3, 6, 6, 0, 1), 3.5) >>> output = m(input) @@ -173,39 +171,32 @@ class ReflectionPad1d(_ReflectionPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 2-`tuple`, uses (`paddingLeft`, `paddingRight`) + padding in all boundaries. If a 2-`tuple`, uses + (:math:`\text{padding\_left}`, :math:`\text{padding\_right}`) Shape: - Input: :math:`(N, C, W_{in})` - Output: :math:`(N, C, W_{out})` where - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ReflectionPad1d(2) - >>> input = torch.arange(8).reshape(1, 2, 4) + >>> input = torch.arange(8, dtype=torch.float).reshape(1, 2, 4) >>> input - - (0 ,.,.) = - 0 1 2 3 - 4 5 6 7 - [torch.FloatTensor of size (1,2,4)] - + tensor([[[0., 1., 2., 3.], + [4., 5., 6., 7.]]]) >>> m(input) - - (0 ,.,.) = - 2 1 0 1 2 3 2 1 - 6 5 4 5 6 7 6 5 - [torch.FloatTensor of size (1,2,8)] - - >>> # using different paddings + tensor([[[2., 1., 0., 1., 2., 3., 2., 1.], + [6., 5., 4., 5., 6., 7., 6., 5.]]]) + >>> m(input) + tensor([[[2., 1., 0., 1., 2., 3., 2., 1.], + [6., 5., 4., 5., 6., 7., 6., 5.]]]) + >>> # using different paddings for different sides >>> m = nn.ReflectionPad1d((3, 1)) >>> m(input) - - (0 ,.,.) = - 3 2 1 0 1 2 3 2 - 7 6 5 4 5 6 7 6 - [torch.FloatTensor of size (1,2,8)] + tensor([[[3., 2., 1., 0., 1., 2., 3., 2.], + [7., 6., 5., 4., 5., 6., 7., 6.]]]) """ @@ -221,50 +212,40 @@ class ReflectionPad2d(_ReflectionPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 4-`tuple`, uses (`paddingLeft`, `paddingRight`, - `paddingTop`, `paddingBottom`) + padding in all boundaries. If a 4-`tuple`, uses (:math:`\text{padding\_left}`, + :math:`\text{padding\_right}`, :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`) Shape: - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ReflectionPad2d(2) - >>> input = torch.arange(9).reshape(1, 1, 3, 3) + >>> input = torch.arange(9, dtype=torch.float).reshape(1, 1, 3, 3) >>> input - - (0 ,0 ,.,.) = - 0 1 2 - 3 4 5 - 6 7 8 - [torch.FloatTensor of size (1,1,3,3)] - + tensor([[[[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]]]]) >>> m(input) - - (0 ,0 ,.,.) = - 8 7 6 7 8 7 6 - 5 4 3 4 5 4 3 - 2 1 0 1 2 1 0 - 5 4 3 4 5 4 3 - 8 7 6 7 8 7 6 - 5 4 3 4 5 4 3 - 2 1 0 1 2 1 0 - [torch.FloatTensor of size (1,1,7,7)] - - >>> # using different paddings + tensor([[[[8., 7., 6., 7., 8., 7., 6.], + [5., 4., 3., 4., 5., 4., 3.], + [2., 1., 0., 1., 2., 1., 0.], + [5., 4., 3., 4., 5., 4., 3.], + [8., 7., 6., 7., 8., 7., 6.], + [5., 4., 3., 4., 5., 4., 3.], + [2., 1., 0., 1., 2., 1., 0.]]]]) + >>> # using different paddings for different sides >>> m = nn.ReflectionPad2d((1, 1, 2, 0)) >>> m(input) - - (0 ,0 ,.,.) = - 7 6 7 8 7 - 4 3 4 5 4 - 1 0 1 2 1 - 4 3 4 5 4 - 7 6 7 8 7 - [torch.FloatTensor of size (1,1,5,5)] + tensor([[[[7., 6., 7., 8., 7.], + [4., 3., 4., 5., 4.], + [1., 0., 1., 2., 1.], + [4., 3., 4., 5., 4.], + [7., 6., 7., 8., 7.]]]]) """ @@ -289,39 +270,29 @@ class ReplicationPad1d(_ReplicationPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 2-`tuple`, uses (`paddingLeft`, `paddingRight`) + padding in all boundaries. If a 2-`tuple`, uses + (:math:`\text{padding\_left}`, :math:`\text{padding\_right}`) Shape: - Input: :math:`(N, C, W_{in})` - Output: :math:`(N, C, W_{out})` where - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ReplicationPad1d(2) - >>> input = torch.arange(8).reshape(1, 2, 4) + >>> input = torch.arange(8, dtype=torch.float).reshape(1, 2, 4) >>> input - - (0 ,.,.) = - 0 1 2 3 - 4 5 6 7 - [torch.FloatTensor of size (1,2,4)] - + tensor([[[0., 1., 2., 3.], + [4., 5., 6., 7.]]]) >>> m(input) - - (0 ,.,.) = - 0 0 0 1 2 3 3 3 - 4 4 4 5 6 7 7 7 - [torch.FloatTensor of size (1,2,8)] - - >>> # using different paddings + tensor([[[0., 0., 0., 1., 2., 3., 3., 3.], + [4., 4., 4., 5., 6., 7., 7., 7.]]]) + >>> # using different paddings for different sides >>> m = nn.ReplicationPad1d((3, 1)) >>> m(input) - - (0 ,.,.) = - 0 0 0 0 1 2 3 3 - 4 4 4 4 5 6 7 7 - [torch.FloatTensor of size (1,2,8)] + tensor([[[0., 0., 0., 0., 1., 2., 3., 3.], + [4., 4., 4., 4., 5., 6., 7., 7.]]]) """ @@ -337,50 +308,39 @@ class ReplicationPad2d(_ReplicationPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 4-`tuple`, uses (`paddingLeft`, `paddingRight`, - `paddingTop`, `paddingBottom`) + padding in all boundaries. If a 4-`tuple`, uses (:math:`\text{padding\_left}`, + :math:`\text{padding\_right}`, :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`) Shape: - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ReplicationPad2d(2) - >>> input = torch.arange(9).reshape(1, 1, 3, 3) + >>> input = torch.arange(9, dtype=torch.float).reshape(1, 1, 3, 3) >>> input - - (0 ,0 ,.,.) = - 0 1 2 - 3 4 5 - 6 7 8 - [torch.FloatTensor of size (1,1,3,3)] - + tensor([[[[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]]]]) >>> m(input) - - (0 ,0 ,.,.) = - 0 0 0 1 2 2 2 - 0 0 0 1 2 2 2 - 0 0 0 1 2 2 2 - 3 3 3 4 5 5 5 - 6 6 6 7 8 8 8 - 6 6 6 7 8 8 8 - 6 6 6 7 8 8 8 - [torch.FloatTensor of size (1,1,7,7)] - - >>> # using different paddings + tensor([[[[0., 0., 0., 1., 2., 2., 2.], + [0., 0., 0., 1., 2., 2., 2.], + [0., 0., 0., 1., 2., 2., 2.], + [3., 3., 3., 4., 5., 5., 5.], + [6., 6., 6., 7., 8., 8., 8.], + [6., 6., 6., 7., 8., 8., 8.], + [6., 6., 6., 7., 8., 8., 8.]]]]) + >>> # using different paddings for different sides >>> m = nn.ReplicationPad2d((1, 1, 2, 0)) >>> m(input) - - (0 ,0 ,.,.) = - 0 0 1 2 2 - 0 0 1 2 2 - 0 0 1 2 2 - 3 3 4 5 5 - 6 6 7 8 8 - [torch.FloatTensor of size (1,1,5,5)] + tensor([[[[0., 0., 1., 2., 2.], + [0., 0., 1., 2., 2.], + [0., 0., 1., 2., 2.], + [3., 3., 4., 5., 5.], + [6., 6., 7., 8., 8.]]]]) """ @@ -396,22 +356,24 @@ class ReplicationPad3d(_ReplicationPadNd): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 6-`tuple`, uses (`paddingLeft`, `paddingRight`, - `paddingTop`, `paddingBottom`, `paddingFront`, `paddingBack`) + padding in all boundaries. If a 6-`tuple`, uses + (:math:`\text{padding\_left}`, :math:`\text{padding\_right}`, + :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`, + :math:`\text{padding\_front}`, :math:`\text{padding\_back}`) Shape: - Input: :math:`(N, C, D_{in}, H_{in}, W_{in})` - Output: :math:`(N, C, D_{out}, H_{out}, W_{out})` where - :math:`D_{out} = D_{in} + \textit{paddingFront} + \textit{paddingBack}` - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`D_{out} = D_{in} + \text{padding\_front} + \text{padding\_back}` + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ReplicationPad3d(3) >>> input = torch.randn(16, 3, 8, 320, 480) >>> output = m(input) - >>> # using different paddings + >>> # using different paddings for different sides >>> m = nn.ReplicationPad3d((3, 3, 6, 6, 1, 1)) >>> output = m(input) @@ -429,50 +391,39 @@ class ZeroPad2d(ConstantPad2d): Args: padding (int, tuple): the size of the padding. If is `int`, uses the same - padding in all boundaries. If a 4-`tuple`, uses (`paddingLeft`, `paddingRight`, - `paddingTop`, `paddingBottom`) + padding in all boundaries. If a 4-`tuple`, uses (:math:`\text{padding\_left}`, + :math:`\text{padding\_right}`, :math:`\text{padding\_top}`, :math:`\text{padding\_bottom}`) Shape: - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - :math:`H_{out} = H_{in} + \textit{paddingTop} + \textit{paddingBottom}` - :math:`W_{out} = W_{in} + \textit{paddingLeft} + \textit{paddingRight}` + :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}` + :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}` Examples:: >>> m = nn.ZeroPad2d(2) >>> input = torch.randn(1, 1, 3, 3) >>> input - - (0 ,0 ,.,.) = - 1.4418 -1.9812 -0.3815 - -0.3828 -0.6833 -0.2376 - 0.1433 0.0211 0.4311 - [torch.FloatTensor of size (1,1,3,3)] - + tensor([[[[-0.1678, -0.4418, 1.9466], + [ 0.9604, -0.4219, -0.5241], + [-0.9162, -0.5436, -0.6446]]]]) >>> m(input) - - (0 ,0 ,.,.) = - 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 - 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 - 0.0000 0.0000 1.4418 -1.9812 -0.3815 0.0000 0.0000 - 0.0000 0.0000 -0.3828 -0.6833 -0.2376 0.0000 0.0000 - 0.0000 0.0000 0.1433 0.0211 0.4311 0.0000 0.0000 - 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 - 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 - [torch.FloatTensor of size (1,1,7,7)] - - >>> # using different paddings + tensor([[[[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, 0.0000, -0.1678, -0.4418, 1.9466, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.9604, -0.4219, -0.5241, 0.0000, 0.0000], + [ 0.0000, 0.0000, -0.9162, -0.5436, -0.6446, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000]]]]) + >>> # using different paddings for different sides >>> m = nn.ZeroPad2d((1, 1, 2, 0)) >>> m(input) - - (0 ,0 ,.,.) = - 0.0000 0.0000 0.0000 0.0000 0.0000 - 0.0000 0.0000 0.0000 0.0000 0.0000 - 0.0000 1.4418 -1.9812 -0.3815 0.0000 - 0.0000 -0.3828 -0.6833 -0.2376 0.0000 - 0.0000 0.1433 0.0211 0.4311 0.0000 - [torch.FloatTensor of size (1,1,5,5)] + tensor([[[[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, -0.1678, -0.4418, 1.9466, 0.0000], + [ 0.0000, 0.9604, -0.4219, -0.5241, 0.0000], + [ 0.0000, -0.9162, -0.5436, -0.6446, 0.0000]]]]) """ diff --git a/torch/nn/modules/pixelshuffle.py b/torch/nn/modules/pixelshuffle.py index 26c325f72f631..e68946d7c7fe9 100644 --- a/torch/nn/modules/pixelshuffle.py +++ b/torch/nn/modules/pixelshuffle.py @@ -17,8 +17,8 @@ class PixelShuffle(Module): upscale_factor (int): factor to increase spatial resolution by Shape: - - Input: :math:`(N, C * \text{upscale_factor}^2, H, W)` - - Output: :math:`(N, C, H * \text{upscale_factor}, W * \text{upscale_factor})` + - Input: :math:`(N, C * \text{upscale\_factor}^2, H, W)` + - Output: :math:`(N, C, H * \text{upscale\_factor}, W * \text{upscale\_factor})` Examples:: diff --git a/torch/nn/modules/pooling.py b/torch/nn/modules/pooling.py index 75ab843e5413b..7ca95d21230ca 100644 --- a/torch/nn/modules/pooling.py +++ b/torch/nn/modules/pooling.py @@ -30,11 +30,8 @@ class MaxPool1d(_MaxPoolNd): and output :math:`(N, C, L_{out})` can be precisely described as: .. math:: - - \begin{equation*} - \text{out}(N_i, C_j, k) = \max_{m=0, \ldots, \text{kernel_size}-1} - \text{input}(N_i, C_j, \text{stride} * k + m) - \end{equation*} + out(N_i, C_j, k) = \max_{m=0, \ldots, kernel\_size-1} + input(N_i, C_j, stride * k + m) If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides for :attr:`padding` number of points. :attr:`dilation` controls the spacing between the kernel points. @@ -55,7 +52,7 @@ class MaxPool1d(_MaxPoolNd): .. math:: L_{out} = \left\lfloor \frac{L_{in} + 2 * \text{padding} - \text{dilation} - * (\text{kernel_size} - 1) - 1}{\text{stride}} + 1\right\rfloor + * (\text{kernel\_size} - 1) - 1}{\text{stride}} + 1\right\rfloor Examples:: @@ -88,10 +85,8 @@ class MaxPool2d(_MaxPoolNd): .. math:: - \begin{equation*} - \text{out}(N_i, C_j, h, w) = \max_{m=0, \ldots, kH-1} \max_{n=0, \ldots, kW-1} - \text{input}(N_i, C_j, \text{stride}[0] * h + m, \text{stride}[1] * w + n) - \end{equation*} + out(N_i, C_j, h, w) = \max_{m=0, \ldots, kH-1} \max_{n=0, \ldots, kW-1} + \text{input}(N_i, C_j, \text{stride[0]} * h + m, \text{stride[1]} * w + n) If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides for :attr:`padding` number of points. :attr:`dilation` controls the spacing between the kernel points. @@ -117,11 +112,12 @@ class MaxPool2d(_MaxPoolNd): - Output: :math:`(N, C, H_{out}, W_{out})` where .. math:: - H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding}[0] - \text{dilation}[0] - * (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding[0]} - \text{dilation[0]} + * (\text{kernel\_size[0]} - 1) - 1}{\text{stride[0]}} + 1\right\rfloor - W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding}[1] - \text{dilation}[1] - * (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + .. math:: + W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding[1]} - \text{dilation[1]} + * (\text{kernel\_size[1]} - 1) - 1}{\text{stride[1]}} + 1\right\rfloor Examples:: @@ -144,18 +140,21 @@ def forward(self, input): class MaxPool3d(_MaxPoolNd): r"""Applies a 3D max pooling over an input signal composed of several input - planes. + planes. This is not a test In the simplest case, the output value of the layer with input size :math:`(N, C, D, H, W)`, output :math:`(N, C, D_{out}, H_{out}, W_{out})` and :attr:`kernel_size` :math:`(kD, kH, kW)` can be precisely described as: .. math:: - - \begin{align*} - \text{out}(N_i, C_j, d, h, w) &= \max_{k=0, \ldots, kD-1} \max_{m=0, \ldots, kH-1} \max_{n=0, \ldots, kW-1} - \text{input}(N_i, C_j, \text{stride}[0] * k + d,\\ &\text{stride}[1] * h + m, \text{stride}[2] * w + n) - \end{align*} + out(N_i, C_j, d, h, w) = + \begin{gathered} + \max_{k=0, \ldots, kD-1} + \max_{m=0, \ldots, kH-1} + \max_{n=0, \ldots, kW-1} \\ + \text{input}(N_i, C_j, \text{stride[0]} * + k + d, \text{stride[1]} * h + m, \text{stride[2]} * w + n) + \end{gathered} If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides for :attr:`padding` number of points. :attr:`dilation` controls the spacing between the kernel points. @@ -182,13 +181,15 @@ class MaxPool3d(_MaxPoolNd): .. math:: D_{out} = \left\lfloor\frac{D_{in} + 2 * \text{padding}[0] - \text{dilation}[0] * - (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + .. math:: H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding}[1] - \text{dilation}[1] * - (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + .. math:: W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding}[2] - \text{dilation}[2] * - (\text{kernel_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor + (\text{kernel\_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor Examples:: @@ -248,7 +249,7 @@ class MaxUnpool1d(_MaxUnpoolNd): - Output: :math:`(N, C, H_{out})` where .. math:: - H_{out} = (H_{in} - 1) * \text{stride}[0] - 2 * \text{padding}[0] + \text{kernel_size}[0] + H_{out} = (H_{in} - 1) * \text{stride}[0] - 2 * \text{padding}[0] + \text{kernel\_size}[0] or as given by :attr:`output_size` in the call operator @@ -313,9 +314,10 @@ class MaxUnpool2d(_MaxUnpoolNd): - Output: :math:`(N, C, H_{out}, W_{out})` where .. math:: - H_{out} = (H_{in} - 1) * \text{stride}[0] - 2 * \text{padding}[0] + \text{kernel_size}[0] + H_{out} = (H_{in} - 1) * \text{stride[0]} - 2 * \text{padding[0]} + \text{kernel\_size[0]} - W_{out} = (W_{in} - 1) * \text{stride}[1] - 2 * \text{padding}[1] + \text{kernel_size}[1] + .. math:: + W_{out} = (W_{in} - 1) * \text{stride[1]} - 2 * \text{padding[1]} + \text{kernel\_size[1]} or as given by :attr:`output_size` in the call operator @@ -384,11 +386,13 @@ class MaxUnpool3d(_MaxUnpoolNd): - Output: :math:`(N, C, D_{out}, H_{out}, W_{out})` where .. math:: - D_{out} = (D_{in} - 1) * \text{stride}[0] - 2 * \text{padding}[0] + \text{kernel_size}[0] + D_{out} = (D_{in} - 1) * \text{stride[0]} - 2 * \text{padding[0]} + \text{kernel\_size[0]} - H_{out} = (H_{in} - 1) * \text{stride}[1] - 2 * \text{padding}[1] + \text{kernel_size}[1] + .. math:: + H_{out} = (H_{in} - 1) * \text{stride[1]} - 2 * \text{padding[1]} + \text{kernel\_size[1]} - W_{out} = (W_{in} - 1) * \text{stride}[2] - 2 * \text{padding}[2] + \text{kernel_size}[2] + .. math:: + W_{out} = (W_{in} - 1) * \text{stride[2]} - 2 * \text{padding[2]} + \text{kernel\_size[2]} or as given by :attr:`output_size` in the call operator @@ -432,10 +436,8 @@ class AvgPool1d(_AvgPoolNd): .. math:: - \begin{equation*} \text{out}(N_i, C_j, l) = \frac{1}{k} \sum_{m=0}^{k} \text{input}(N_i, C_j, \text{stride} * l + m) - \end{equation*} If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides for :attr:`padding` number of points. @@ -456,7 +458,7 @@ class AvgPool1d(_AvgPoolNd): .. math:: L_{out} = \left\lfloor \frac{L_{in} + - 2 * \text{padding} - \text{kernel_size}}{\text{stride}} + 1\right\rfloor + 2 * \text{padding} - \text{kernel\_size}}{\text{stride}} + 1\right\rfloor Examples:: @@ -491,10 +493,8 @@ class AvgPool2d(_AvgPoolNd): .. math:: - \begin{equation*} - \text{out}(N_i, C_j, h, w) = \frac{1}{kH * kW} \sum_{m=0}^{kH-1} \sum_{n=0}^{kW-1} - \text{input}(N_i, C_j, \text{stride}[0] * h + m, \text{stride}[1] * w + n) - \end{equation*} + out(N_i, C_j, h, w) = \frac{1}{kH * kW} \sum_{m=0}^{kH-1} \sum_{n=0}^{kW-1} + input(N_i, C_j, stride[0] * h + m, stride[1] * w + n) If :attr:`padding` is non-zero, then the input is implicitly zero-padded on both sides for :attr:`padding` number of points. @@ -518,10 +518,11 @@ class AvgPool2d(_AvgPoolNd): .. math:: H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding}[0] - - \text{kernel_size}[0]}{\text{stride}[0]} + 1\right\rfloor + \text{kernel\_size}[0]}{\text{stride}[0]} + 1\right\rfloor + .. math:: W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding}[1] - - \text{kernel_size}[1]}{\text{stride}[1]} + 1\right\rfloor + \text{kernel\_size}[1]}{\text{stride}[1]} + 1\right\rfloor Examples:: @@ -557,12 +558,10 @@ class AvgPool3d(_AvgPoolNd): .. math:: - \begin{equation*} \text{out}(N_i, C_j, d, h, w) = \sum_{k=0}^{kD-1} \sum_{m=0}^{kH-1} \sum_{n=0}^{kW-1} \frac{\text{input}(N_i, C_j, \text{stride}[0] * d + k, \text{stride}[1] * h + m, \text{stride}[2] * w + n)} {kD * kH * kW} - \end{equation*} If :attr:`padding` is non-zero, then the input is implicitly zero-padded on all three sides for :attr:`padding` number of points. @@ -586,13 +585,15 @@ class AvgPool3d(_AvgPoolNd): .. math:: D_{out} = \left\lfloor\frac{D_{in} + 2 * \text{padding}[0] - - \text{kernel_size}[0]}{\text{stride}[0]} + 1\right\rfloor + \text{kernel\_size}[0]}{\text{stride}[0]} + 1\right\rfloor + .. math:: H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding}[1] - - \text{kernel_size}[1]}{\text{stride}[1]} + 1\right\rfloor + \text{kernel\_size}[1]}{\text{stride}[1]} + 1\right\rfloor + .. math:: W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding}[2] - - \text{kernel_size}[2]}{\text{stride}[2]} + 1\right\rfloor + \text{kernel\_size}[2]}{\text{stride}[2]} + 1\right\rfloor Examples:: @@ -721,7 +722,7 @@ class LPPool1d(_LPPoolNd): .. math:: L_{out} = \left\lfloor\frac{L_{in} + - 2 * \text{padding} - \text{kernel_size}}{\text{stride}} + 1\right\rfloor + 2 * \text{padding} - \text{kernel\_size}}{\text{stride}} + 1\right\rfloor Examples:: >>> # power-2 pool of window of length 3, with stride 2. @@ -767,10 +768,11 @@ class LPPool2d(_LPPoolNd): .. math:: H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding}[0] - \text{dilation}[0] * - (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + .. math:: W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding}[1] - \text{dilation}[1] * - (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor Examples:: diff --git a/torch/nn/modules/rnn.py b/torch/nn/modules/rnn.py index 9f726d12e7a72..45d5a216ff332 100644 --- a/torch/nn/modules/rnn.py +++ b/torch/nn/modules/rnn.py @@ -235,7 +235,7 @@ def all_weights(self): class RNN(RNNBase): - r"""Applies a multi-layer Elman RNN with `tanh` or `ReLU` non-linearity to an + r"""Applies a multi-layer Elman RNN with :math:`tanh` or :math:`ReLU` non-linearity to an input sequence. @@ -243,8 +243,7 @@ class RNN(RNNBase): function: .. math:: - - h_t = \tanh(w_{ih} x_t + b_{ih} + w_{hh} h_{(t-1)} + b_{hh}) + h_t = \text{tanh}(w_{ih} x_t + b_{ih} + w_{hh} h_{(t-1)} + b_{hh}) where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the input at time `t`, and :math:`h_{(t-1)}` is the hidden state of the @@ -307,7 +306,7 @@ class RNN(RNNBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: @@ -342,15 +341,14 @@ class LSTM(RNNBase): function: .. math:: - - \begin{array}{ll} + \begin{array}{ll} \\ i_t = \sigma(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\ f_t = \sigma(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\ g_t = \tanh(W_{ig} x_t + b_{ig} + W_{hg} h_{(t-1)} + b_{hg}) \\ o_t = \sigma(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\ c_t = f_t c_{(t-1)} + i_t g_t \\ - h_t = o_t \tanh(c_t) - \end{array} + h_t = o_t \tanh(c_t) \\ + \end{array} where :math:`h_t` is the hidden state at time `t`, :math:`c_t` is the cell state at time `t`, :math:`x_t` is the input at time `t`, :math:`h_{(t-1)}` @@ -419,7 +417,7 @@ class LSTM(RNNBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: @@ -442,13 +440,12 @@ class GRU(RNNBase): function: .. math:: - - \begin{array}{ll} + \begin{array}{ll} r_t = \sigma(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\ z_t = \sigma(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\ n_t = \tanh(W_{in} x_t + b_{in} + r_t (W_{hn} h_{(t-1)}+ b_{hn})) \\ - h_t = (1 - z_t) n_t + z_t h_{(t-1)} \\ - \end{array} + h_t = (1 - z_t) n_t + z_t h_{(t-1)} + \end{array} where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the input at time `t`, :math:`h_{(t-1)}` is the hidden state of the previous layer @@ -509,7 +506,7 @@ class GRU(RNNBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: @@ -587,7 +584,7 @@ class RNNCell(RNNCellBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: @@ -688,7 +685,7 @@ class LSTMCell(RNNCellBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: @@ -776,7 +773,7 @@ class GRUCell(RNNCellBase): .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{hidden_size}}` + where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: diff --git a/torch/nn/modules/upsampling.py b/torch/nn/modules/upsampling.py index 5e0e6671374a9..e416326229a14 100644 --- a/torch/nn/modules/upsampling.py +++ b/torch/nn/modules/upsampling.py @@ -33,12 +33,14 @@ class Upsample(Module): - Output: :math:`(N, C, W_{out})`, :math:`(N, C, H_{out}, W_{out})` or :math:`(N, C, D_{out}, H_{out}, W_{out})`, where - .. math:: - D_{out} = \left\lfloor D_{in} \times \text{scale_factor} \right\rfloor \text{ or size}[-3] + .. math:: + D_{out} = \left\lfloor D_{in} \times \text{scale\_factor} \right\rfloor \text{ or size}[-3] - H_{out} = \left\lfloor H_{in} \times \text{scale_factor} \right\rfloor \text{ or size}[-2] + .. math:: + H_{out} = \left\lfloor H_{in} \times \text{scale\_factor} \right\rfloor \text{ or size}[-2] - W_{out} = \left\lfloor W_{in} \times \text{scale_factor} \right\rfloor \text{ or size}[-1] + .. math:: + W_{out} = \left\lfloor W_{in} \times \text{scale\_factor} \right\rfloor \text{ or size}[-1] .. warning:: With ``align_corners = True``, the linearly interpolating modes @@ -151,10 +153,11 @@ class UpsamplingNearest2d(Upsample): - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - .. math:: - H_{out} = \left\lfloor H_{in} \times \text{scale_factor} \right\rfloor + .. math:: + H_{out} = \left\lfloor H_{in} \times \text{scale\_factor} \right\rfloor - W_{out} = \left\lfloor W_{in} \times \text{scale_factor} \right\rfloor + .. math:: + W_{out} = \left\lfloor W_{in} \times \text{scale\_factor} \right\rfloor Examples:: @@ -199,10 +202,11 @@ class UpsamplingBilinear2d(Upsample): - Input: :math:`(N, C, H_{in}, W_{in})` - Output: :math:`(N, C, H_{out}, W_{out})` where - .. math:: - H_{out} = \left\lfloor H_{in} \times \text{scale_factor} \right\rfloor + .. math:: + H_{out} = \left\lfloor H_{in} \times \text{scale\_factor} \right\rfloor - W_{out} = \left\lfloor W_{in} \times \text{scale_factor} \right\rfloor + .. math:: + W_{out} = \left\lfloor W_{in} \times \text{scale\_factor} \right\rfloor Examples:: diff --git a/torch/nn/utils/spectral_norm.py b/torch/nn/utils/spectral_norm.py index 1cdafb39e3830..bac43badbabf3 100644 --- a/torch/nn/utils/spectral_norm.py +++ b/torch/nn/utils/spectral_norm.py @@ -81,8 +81,8 @@ def spectral_norm(module, name='weight', n_power_iterations=1, eps=1e-12, dim=No r"""Applies spectral normalization to a parameter in the given module. .. math:: - \mathbf{W} &= \dfrac{\mathbf{W}}{\sigma(\mathbf{W})} \\ - \sigma(\mathbf{W}) &= \max_{\mathbf{h}: \mathbf{h} \ne 0} \dfrac{\|\mathbf{W} \mathbf{h}\|_2}{\|\mathbf{h}\|_2} + \mathbf{W} = \dfrac{\mathbf{W}}{\sigma(\mathbf{W})} \\ + \sigma(\mathbf{W}) = \max_{\mathbf{h}: \mathbf{h} \ne 0} \dfrac{\|\mathbf{W} \mathbf{h}\|_2}{\|\mathbf{h}\|_2} Spectral normalization stabilizes the training of discriminators (critics) in Generaive Adversarial Networks (GANs) by rescaling the weight tensor