I am done

rl/Lib/site-packages/torch/include/caffe2/serialize/file_adapter.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <fstream>
+#include <memory>
+#include <c10/macros/Macros.h>
+
+#include "caffe2/serialize/istream_adapter.h"
+#include "caffe2/serialize/read_adapter_interface.h"
+
+namespace caffe2 {
+namespace serialize {
+
+class TORCH_API FileAdapter final : public ReadAdapterInterface {
+ public:
+  C10_DISABLE_COPY_AND_ASSIGN(FileAdapter);
+  explicit FileAdapter(const std::string& file_name);
+  size_t size() const override;
+  size_t read(uint64_t pos, void* buf, size_t n, const char* what = "")
+      const override;
+  ~FileAdapter() override;
+
+ private:
+  // An RAII Wrapper for a FILE pointer. Closes on destruction.
+  struct RAIIFile {
+    FILE* fp_;
+    explicit RAIIFile(const std::string& file_name);
+    ~RAIIFile();
+  };
+
+  RAIIFile file_;
+  // The size of the opened file in bytes
+  uint64_t size_;
+};
+
+} // namespace serialize
+} // namespace caffe2

rl/Lib/site-packages/torch/include/caffe2/serialize/in_memory_adapter.h

@@ -0,0 +1,32 @@
+#pragma once
+#include <cstring>
+#include <caffe2/serialize/read_adapter_interface.h>
+
+
+namespace caffe2 {
+namespace serialize {
+
+class MemoryReadAdapter final : public caffe2::serialize::ReadAdapterInterface {
+ public:
+  explicit MemoryReadAdapter(const void* data, off_t size)
+      : data_(data), size_(size) {}
+
+  size_t size() const override {
+    return size_;
+  }
+
+  size_t read(uint64_t pos, void* buf, size_t n, const char* what = "")
+      const override {
+    (void) what;
+    memcpy(buf, (int8_t*)(data_) + pos, n);
+    return n;
+  }
+
+ private:
+  const void* data_;
+  off_t size_;
+};
+
+
+} // namespace serialize
+} // namespace caffe2

rl/Lib/site-packages/torch/include/caffe2/serialize/inline_container.h

@@ -0,0 +1,278 @@
+#pragma once
+
+#include <cerrno>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <istream>
+#include <mutex>
+#include <ostream>
+#include <unordered_set>
+
+#include <c10/core/Allocator.h>
+#include <c10/core/Backend.h>
+
+#include "caffe2/serialize/istream_adapter.h"
+#include "caffe2/serialize/read_adapter_interface.h"
+#include "caffe2/serialize/versions.h"
+
+
+extern "C" {
+typedef struct mz_zip_archive mz_zip_archive;
+}
+
+// PyTorch containers are a special zip archive with the following layout
+// archive_name.zip contains:
+//    archive_name/
+//        version # a file with a single decimal number written in ascii,
+//                # used to establish the version of the archive format
+//        model.json # overall model description, this is a json output of
+//                   # ModelDef from torch.proto
+//        # the following names are by convention only, model.json will
+//        # refer to these files by full names
+//        tensors/
+//          0 # flat storage for tensor data, meta-data about shapes, etc. is
+//            # in model.json
+//          1
+//          ...
+//        # code entries will only exist for modules that have methods attached
+//        code/
+//          archive_name.py # serialized torch script code (python syntax, using
+//          PythonPrint) archive_name_my_submodule.py # submodules have separate
+//          files
+//
+// The PyTorchStreamWriter also ensures additional useful properties for these
+// files
+// 1. All files are stored uncompressed.
+// 2. All files in the archive are aligned to 64 byte boundaries such that
+//    it is possible to mmap the entire file and get an aligned pointer to
+//    tensor data.
+// 3. We universally write in ZIP64 format for consistency.
+
+// The PyTorchStreamReader also provides additional properties:
+// 1. It can read zip files that are created with common
+//    zip tools. This means that even though our writer doesn't compress files,
+//    the reader can still read files that were compressed.
+// 2. It provides a getRecordOffset function which returns the offset into the
+//    raw file where file data lives. If the file was written with
+//    PyTorchStreamWriter it is guaranteed to be 64 byte aligned.
+
+// PyTorchReader/Writer handle checking the version number on the archive format
+// and ensure that all files are written to a archive_name directory so they
+// unzip cleanly.
+
+// When developing this format we want to pay particular attention to the
+// following use cases:
+//
+// -- Reading --
+// 1) Reading with full random access
+//   a) Reading with file api's such as fread()
+//   b) mmaping the file and jumping around the mapped region
+// 2) Reading with 1-pass sequential access
+//      -> A reader will need to build up a data structure of parsed structures
+//         as it reads
+//
+// -- Writing --
+// 1) Writing with full random access
+// 2) Writing with 1-pass sequential access
+//      -> We must take care not to require updating values that have already
+//         been written. We place the variable-length index at the end and do
+//         not put any indicies into the header to fulfill this constraint.
+
+// The model.json, which contains all the metadata information,
+// should be written as the last file. One reason is that the size of tensor
+// data is usually stable. As long as the shape and type of the tensor do not
+// change, the size of the data won't change. On the other sied, the size of the
+// serialized model is likely to change, so we store it as the last record, and
+// we don't need to move previous records when updating the model data.
+
+// The zip format is sufficiently flexible to handle the above use-case.
+// it puts its central directory at the end of the archive and we write
+// model.json as the last file when writing after we have accumulated all
+// other information.
+
+namespace caffe2 {
+namespace serialize {
+
+static constexpr const char* kSerializationIdRecordName = ".data/serialization_id";
+
+struct MzZipReaderIterWrapper;
+
+class TORCH_API ChunkRecordIterator {
+ public:
+  ~ChunkRecordIterator();
+
+  // Read at most `chunkSize` into `buf`. Return the number of actual bytes read.
+  size_t next(void* buf);
+  size_t recordSize() const { return recordSize_; }
+
+ private:
+ ChunkRecordIterator(
+      size_t recordSize,
+      size_t chunkSize,
+      std::unique_ptr<MzZipReaderIterWrapper> iter);
+
+  const size_t recordSize_;
+  const size_t chunkSize_;
+  size_t offset_;
+  std::unique_ptr<MzZipReaderIterWrapper> iter_;
+
+  friend class PyTorchStreamReader;
+};
+
+class TORCH_API PyTorchStreamReader final {
+ public:
+  explicit PyTorchStreamReader(const std::string& file_name);
+  explicit PyTorchStreamReader(std::istream* in);
+  explicit PyTorchStreamReader(std::shared_ptr<ReadAdapterInterface> in);
+
+  // return dataptr, size
+  std::tuple<at::DataPtr, size_t> getRecord(const std::string& name);
+  // multi-thread getRecord
+  std::tuple<at::DataPtr, size_t> getRecord(const std::string& name, std::vector<std::shared_ptr<ReadAdapterInterface>>& additionalReaders);
+  // inplace memory writing
+  size_t getRecord(const std::string& name, void* dst, size_t n);
+  // inplace memory writing, multi-threads.
+  // When additionalReaders is empty, the default behavior is call getRecord(name, dst, n) with default reader
+  // This approach can be used for reading large tensors.
+  size_t getRecord(const std::string& name, void* dst, size_t n,
+    std::vector<std::shared_ptr<ReadAdapterInterface>>& additionalReaders);
+  size_t getRecord(
+      const std::string& name,
+      void* dst,
+      size_t n,
+      size_t chunk_size,
+      void* buf,
+      const std::function<void(void*, const void*, size_t)>& memcpy_func = nullptr);
+
+  // Concurrent reading records with multiple readers.
+  // additionalReaders are additional clients to access the underlying record at different offsets
+  // and write to different trunks of buffers.
+  // If the overall size of the tensor is 10, and size of additionalReader is 2.
+  // The default thread will read [0,4), the additional reader will read [4,8).
+  // The default reader will read [8,10).
+  // The default reader will write to buffer[0,4), the additional reader will write to buffer[4,8),
+  // the additional reader will write to buffer[8,10).
+  // When additionalReaders is empty, the default behavior is call getRecord(name) with default reader
+  // This approach can be used for reading large tensors.
+  size_t getRecordMultiReaders(const std::string& name,
+  std::vector<std::shared_ptr<ReadAdapterInterface>>& additionalReaders,
+  void *dst, size_t n);
+
+  size_t getRecordSize(const std::string& name);
+
+  size_t getRecordOffset(const std::string& name);
+  bool hasRecord(const std::string& name);
+  std::vector<std::string> getAllRecords();
+
+  ChunkRecordIterator createChunkReaderIter(
+      const std::string& name,
+      const size_t recordSize,
+      const size_t chunkSize);
+
+  ~PyTorchStreamReader();
+  uint64_t version() const {
+    return version_;
+  }
+  const std::string& serializationId() {
+    return serialization_id_;
+  }
+
+  void setShouldLoadDebugSymbol(bool should_load_debug_symbol) {
+    load_debug_symbol_ = should_load_debug_symbol;
+  }
+  void setAdditionalReaderSizeThreshold(const size_t& size){
+    additional_reader_size_threshold_ = size;
+  }
+ private:
+  void init();
+  size_t read(uint64_t pos, char* buf, size_t n);
+  void valid(const char* what, const char* info = "");
+  size_t getRecordID(const std::string& name);
+
+  friend size_t
+  istream_read_func(void* pOpaque, uint64_t file_ofs, void* pBuf, size_t n);
+  std::unique_ptr<mz_zip_archive> ar_;
+  std::string archive_name_;
+  std::string archive_name_plus_slash_;
+  std::shared_ptr<ReadAdapterInterface> in_;
+  int64_t version_;
+  std::mutex reader_lock_;
+  bool load_debug_symbol_ = true;
+  std::string serialization_id_;
+  size_t additional_reader_size_threshold_;
+};
+
+class TORCH_API PyTorchStreamWriter final {
+ public:
+  explicit PyTorchStreamWriter(const std::string& archive_name);
+  explicit PyTorchStreamWriter(
+      const std::function<size_t(const void*, size_t)> writer_func);
+
+  void setMinVersion(const uint64_t version);
+
+  void writeRecord(
+      const std::string& name,
+      const void* data,
+      size_t size,
+      bool compress = false);
+  void writeEndOfFile();
+
+  const std::unordered_set<std::string>& getAllWrittenRecords();
+
+  bool finalized() const {
+    return finalized_;
+  }
+
+  const std::string& archiveName() {
+    return archive_name_;
+  }
+
+  const std::string& serializationId() {
+    return serialization_id_;
+  }
+
+  ~PyTorchStreamWriter();
+
+ private:
+  void setup(const std::string& file_name);
+  void valid(const char* what, const char* info = "");
+  void writeSerializationId();
+  size_t current_pos_ = 0;
+  std::unordered_set<std::string> files_written_;
+  std::unique_ptr<mz_zip_archive> ar_;
+  std::string archive_name_;
+  std::string archive_name_plus_slash_;
+  std::string padding_;
+  std::ofstream file_stream_;
+  std::function<size_t(const void*, size_t)> writer_func_;
+  uint64_t combined_uncomp_crc32_ = 0;
+  std::string serialization_id_;
+
+  // This number will be updated when the model has operators
+  // that have valid upgraders.
+  uint64_t version_ = kMinProducedFileFormatVersion;
+  bool finalized_ = false;
+  bool err_seen_ = false;
+  friend size_t ostream_write_func(
+      void* pOpaque,
+      uint64_t file_ofs,
+      const void* pBuf,
+      size_t n);
+};
+
+namespace detail {
+// Writer-specific constants
+constexpr uint64_t kFieldAlignment = 64;
+
+// Returns a record to be appended to the local user extra data entry in order
+// to make data beginning aligned at kFieldAlignment bytes boundary.
+size_t getPadding(
+    size_t cursor,
+    size_t filename_size,
+    size_t size,
+    std::string& padding_buf);
+} // namespace detail
+
+} // namespace serialize
+} // namespace caffe2

rl/Lib/site-packages/torch/include/caffe2/serialize/istream_adapter.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <istream>
+
+#include "c10/macros/Macros.h"
+#include "caffe2/serialize/read_adapter_interface.h"
+
+namespace caffe2 {
+namespace serialize {
+
+// this is a reader implemented by std::istream
+class TORCH_API IStreamAdapter final : public ReadAdapterInterface {
+ public:
+  C10_DISABLE_COPY_AND_ASSIGN(IStreamAdapter);
+  explicit IStreamAdapter(std::istream* istream);
+  size_t size() const override;
+  size_t read(uint64_t pos, void* buf, size_t n, const char* what = "")
+      const override;
+  ~IStreamAdapter() override;
+
+ private:
+  std::istream* istream_;
+  void validate(const char* what) const;
+};
+
+} // namespace serialize
+} // namespace caffe2

rl/Lib/site-packages/torch/include/caffe2/serialize/read_adapter_interface.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+#include "c10/macros/Macros.h"
+
+namespace caffe2 {
+namespace serialize {
+
+// this is the interface for the (file/stream/memory) reader in
+// PyTorchStreamReader. with this interface, we can extend the support
+// besides standard istream
+class TORCH_API ReadAdapterInterface {
+ public:
+  virtual size_t size() const = 0;
+  virtual size_t read(uint64_t pos, void* buf, size_t n, const char* what = "")
+      const = 0;
+  virtual ~ReadAdapterInterface();
+};
+
+} // namespace serialize
+} // namespace caffe2

rl/Lib/site-packages/torch/include/caffe2/serialize/versions.h

@@ -0,0 +1,133 @@
+#pragma once
+#include <cstdint>
+
+namespace caffe2 {
+namespace serialize {
+
+constexpr uint64_t kMinSupportedFileFormatVersion = 0x1L;
+
+constexpr uint64_t kMaxSupportedFileFormatVersion = 0xAL;
+
+// Versions (i.e. why was the version number bumped?)
+
+// Note [Dynamic Versions and torch.jit.save vs. torch.save]
+//
+// Our versioning scheme has a "produced file format version" which
+// describes how an archive is to be read. The version written in an archive
+// is at least this current produced file format version, but may be greater
+// if it includes certain symbols. We refer to these conditional versions
+// as "dynamic," since they are identified at runtime.
+//
+// Dynamic versioning is useful when an operator's semantics are updated.
+// When using torch.jit.save we want those semantics to be preserved. If
+// we bumped the produced file format version on every change, however,
+// then older versions of PyTorch couldn't read even simple archives, like
+// a single tensor, from newer versions of PyTorch. Instead, we
+// assign dynamic versions to these changes that override the
+// produced file format version as needed. That is, when the semantics
+// of torch.div changed it was assigned dynamic version 4, and when
+// torch.jit.saving modules that use torch.div those archives also have
+// (at least) version 4. This prevents earlier versions of PyTorch
+// from accidentally performing the wrong kind of division. Modules
+// that don't use torch.div or other operators with dynamic versions
+// can write the produced file format version, and these programs will
+// run as expected on earlier versions of PyTorch.
+//
+// While torch.jit.save attempts to preserve operator semantics,
+// torch.save does not. torch.save is analogous to pickling Python, so
+// a function that uses torch.div will have different behavior if torch.saved
+// and torch.loaded across PyTorch versions. From a technical perspective,
+// torch.save ignores dynamic versioning.
+
+// 1. Initial version
+// 2. Removed op_version_set version numbers
+// 3. Added type tags to pickle serialization of container types
+// 4. (Dynamic) Stopped integer division using torch.div
+//      (a versioned symbol preserves the historic behavior of versions 1--3)
+// 5. (Dynamic) Stops torch.full inferring a floating point dtype
+//      when given bool or integer fill values.
+// 6. Write version string to `./data/version` instead of `version`.
+
+// [12/15/2021]
+// kProducedFileFormatVersion is set to 7 from 3 due to a different
+// interpretation of what file format version is.
+// Whenever there is new upgrader introduced,
+// this number should be bumped.
+// The reasons that version is bumped in the past:
+//     1. aten::div is changed at version 4
+//     2. aten::full is changed at version 5
+//     3. torch.package uses version 6
+//     4. Introduce new upgrader design and set the version number to 7
+//        mark this change
+// --------------------------------------------------
+// We describe new operator version bump reasons here:
+// 1) [01/24/2022]
+//     We bump the version number to 8 to update aten::linspace
+//     and aten::linspace.out to error out when steps is not
+//     provided. (see: https://github.com/pytorch/pytorch/issues/55951)
+// 2) [01/30/2022]
+//     Bump the version number to 9 to update aten::logspace and
+//     and aten::logspace.out to error out when steps is not
+//     provided. (see: https://github.com/pytorch/pytorch/issues/55951)
+// 3) [02/11/2022]
+//     Bump the version number to 10 to update aten::gelu and
+//     and aten::gelu.out to support the new approximate kwarg.
+//     (see: https://github.com/pytorch/pytorch/pull/61439)
+constexpr uint64_t kProducedFileFormatVersion = 0xAL;
+
+// Absolute minimum version we will write packages. This
+// means that every package from now on will always be
+// greater than this number.
+constexpr uint64_t kMinProducedFileFormatVersion = 0x3L;
+
+// The version we write when the archive contains bytecode.
+// It must be higher or eq to kProducedFileFormatVersion.
+// Because torchscript changes is likely introduce bytecode change.
+// If kProducedFileFormatVersion is increased, kProducedBytecodeVersion
+// should be increased too. The relationship is:
+// kMaxSupportedFileFormatVersion >= (most likely ==) kProducedBytecodeVersion
+//   >= kProducedFileFormatVersion
+// If a format change is forward compatible (still readable by older
+// executables), we will not increment the version number, to minimize the
+// risk of breaking existing clients. TODO: A better way would be to allow
+// the caller that creates a model to specify a maximum version that its
+// clients can accept.
+// Versions:
+//  0x1L: Initial version
+//  0x2L: (Comment missing)
+//  0x3L: (Comment missing)
+//  0x4L: (update) Added schema to function tuple. Forward-compatible change.
+//  0x5L: (update) Update bytecode is sharing constant tensor files from
+//  torchscript, and only serialize extra tensors that are not in the
+//  torchscript constant table. Also update tensor storage schema adapting to
+//  the unify format, the root key of tensor storage is updated from {index} to
+//  {the_pointer_value_the_tensor.storage}, for example:
+//  `140245072983168.storage` Forward-compatibility change.
+//  0x6L: Implicit opereator versioning using number of specified argument.
+//  Refer to the summary of https://github.com/pytorch/pytorch/pull/56845 for
+//  details.
+//  0x7L: Enable support for operators with default arguments plus out
+//  arguments. Refer. See https://github.com/pytorch/pytorch/pull/63651 for
+//  details.
+//  0x8L: Emit promoted operators as instructions. See
+//  https://github.com/pytorch/pytorch/pull/71662 for details.
+//  0x9L: Change serialization format from pickle to format This version is to
+//  serve migration. v8 pickle and v9 flatbuffer are the same. Refer to the
+//  summary of https://github.com/pytorch/pytorch/pull/75201 for more details.
+constexpr uint64_t kProducedBytecodeVersion = 0x8L;
+
+// static_assert(
+//     kProducedBytecodeVersion >= kProducedFileFormatVersion,
+//     "kProducedBytecodeVersion must be higher or equal to
+//     kProducedFileFormatVersion.");
+
+// Introduce kMinSupportedBytecodeVersion and kMaxSupportedBytecodeVersion
+// for limited backward/forward compatibility support of bytecode. If
+// kMinSupportedBytecodeVersion <= model_version <= kMaxSupportedBytecodeVersion
+// (in loader), we should support this model_version. For example, we provide a
+// wrapper to handle an updated operator.
+constexpr uint64_t kMinSupportedBytecodeVersion = 0x4L;
+constexpr uint64_t kMaxSupportedBytecodeVersion = 0x9L;
+
+} // namespace serialize
+} // namespace caffe2