Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/include/kj/filesystem.h @ 69:33d812a61356

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 17:55:14 -0400
parents
children
rev   line source
jpayne@69 1 // Copyright (c) 2015 Sandstorm Development Group, Inc. and contributors
jpayne@69 2 // Licensed under the MIT License:
jpayne@69 3 //
jpayne@69 4 // Permission is hereby granted, free of charge, to any person obtaining a copy
jpayne@69 5 // of this software and associated documentation files (the "Software"), to deal
jpayne@69 6 // in the Software without restriction, including without limitation the rights
jpayne@69 7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
jpayne@69 8 // copies of the Software, and to permit persons to whom the Software is
jpayne@69 9 // furnished to do so, subject to the following conditions:
jpayne@69 10 //
jpayne@69 11 // The above copyright notice and this permission notice shall be included in
jpayne@69 12 // all copies or substantial portions of the Software.
jpayne@69 13 //
jpayne@69 14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
jpayne@69 15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
jpayne@69 16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
jpayne@69 17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
jpayne@69 18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
jpayne@69 19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
jpayne@69 20 // THE SOFTWARE.
jpayne@69 21
jpayne@69 22 #pragma once
jpayne@69 23
jpayne@69 24 #include "memory.h"
jpayne@69 25 #include "io.h"
jpayne@69 26 #include <inttypes.h>
jpayne@69 27 #include "time.h"
jpayne@69 28 #include "function.h"
jpayne@69 29 #include "hash.h"
jpayne@69 30
jpayne@69 31 KJ_BEGIN_HEADER
jpayne@69 32
jpayne@69 33 namespace kj {
jpayne@69 34
jpayne@69 35 template <typename T>
jpayne@69 36 class Vector;
jpayne@69 37
jpayne@69 38 class PathPtr;
jpayne@69 39
jpayne@69 40 class Path {
jpayne@69 41 // A Path identifies a file in a directory tree.
jpayne@69 42 //
jpayne@69 43 // In KJ, we avoid representing paths as plain strings because this can lead to path injection
jpayne@69 44 // bugs as well as numerous kinds of bugs relating to path parsing edge cases. The Path class's
jpayne@69 45 // interface is designed to "make it hard to screw up".
jpayne@69 46 //
jpayne@69 47 // A "Path" is in fact a list of strings, each string being one component of the path (as would
jpayne@69 48 // normally be separated by '/'s). Path components are not allowed to contain '/' nor '\0', nor
jpayne@69 49 // are they allowed to be the special names "", ".", nor "..".
jpayne@69 50 //
jpayne@69 51 // If you explicitly want to parse a path that contains '/'s, ".", and "..", you must use
jpayne@69 52 // parse() and/or eval(). However, users of this interface are encouraged to avoid parsing
jpayne@69 53 // paths at all, and instead express paths as string arrays.
jpayne@69 54 //
jpayne@69 55 // Note that when using the Path class, ".." is always canonicalized in path space without
jpayne@69 56 // consulting the actual filesystem. This means that "foo/some-symlink/../bar" is exactly
jpayne@69 57 // equivalent to "foo/bar". This differs from the kernel's behavior when resolving paths passed
jpayne@69 58 // to system calls: the kernel would have resolved "some-symlink" to its target physical path,
jpayne@69 59 // and then would have interpreted ".." relative to that. In practice, the kernel's behavior is
jpayne@69 60 // rarely what the user or programmer intended, hence canonicalizing in path space produces a
jpayne@69 61 // better result.
jpayne@69 62 //
jpayne@69 63 // Path objects are "immutable": functions that "modify" the path return a new path. However,
jpayne@69 64 // if the path being operated on is an rvalue, copying can be avoided. Hence it makes sense to
jpayne@69 65 // write code like:
jpayne@69 66 //
jpayne@69 67 // Path p = ...;
jpayne@69 68 // p = kj::mv(p).append("bar"); // in-place update, avoids string copying
jpayne@69 69
jpayne@69 70 public:
jpayne@69 71 Path(decltype(nullptr)); // empty path
jpayne@69 72
jpayne@69 73 explicit Path(StringPtr name);
jpayne@69 74 explicit Path(String&& name);
jpayne@69 75 // Create a Path containing only one component. `name` is a single filename; it cannot contain
jpayne@69 76 // '/' nor '\0' nor can it be exactly "" nor "." nor "..".
jpayne@69 77 //
jpayne@69 78 // If you want to allow '/'s and such, you must call Path::parse(). We force you to do this to
jpayne@69 79 // prevent path injection bugs where you didn't consider what would happen if the path contained
jpayne@69 80 // a '/'.
jpayne@69 81
jpayne@69 82 explicit Path(std::initializer_list<StringPtr> parts);
jpayne@69 83 explicit Path(ArrayPtr<const StringPtr> parts);
jpayne@69 84 explicit Path(Array<String> parts);
jpayne@69 85 // Construct a path from an array. Note that this means you can do:
jpayne@69 86 //
jpayne@69 87 // Path{"foo", "bar", "baz"} // equivalent to Path::parse("foo/bar/baz")
jpayne@69 88
jpayne@69 89 KJ_DISALLOW_COPY(Path);
jpayne@69 90 Path(Path&&) = default;
jpayne@69 91 Path& operator=(Path&&) = default;
jpayne@69 92
jpayne@69 93 Path clone() const;
jpayne@69 94
jpayne@69 95 static Path parse(StringPtr path);
jpayne@69 96 // Parses a path in traditional format. Components are separated by '/'. Any use of "." or
jpayne@69 97 // ".." will be canonicalized (if they can't be canonicalized, e.g. because the path starts with
jpayne@69 98 // "..", an exception is thrown). Multiple consecutive '/'s will be collapsed. A leading '/'
jpayne@69 99 // is NOT accepted -- if that is a problem, you probably want `eval()`. Trailing '/'s are
jpayne@69 100 // ignored.
jpayne@69 101
jpayne@69 102 Path append(Path&& suffix) const&;
jpayne@69 103 Path append(Path&& suffix) &&;
jpayne@69 104 Path append(PathPtr suffix) const&;
jpayne@69 105 Path append(PathPtr suffix) &&;
jpayne@69 106 Path append(StringPtr suffix) const&;
jpayne@69 107 Path append(StringPtr suffix) &&;
jpayne@69 108 Path append(String&& suffix) const&;
jpayne@69 109 Path append(String&& suffix) &&;
jpayne@69 110 // Create a new path by appending the given path to this path.
jpayne@69 111 //
jpayne@69 112 // `suffix` cannot contain '/' characters. Instead, you can append an array:
jpayne@69 113 //
jpayne@69 114 // path.append({"foo", "bar"})
jpayne@69 115 //
jpayne@69 116 // Or, use Path::parse():
jpayne@69 117 //
jpayne@69 118 // path.append(Path::parse("foo//baz/../bar"))
jpayne@69 119
jpayne@69 120 Path eval(StringPtr pathText) const&;
jpayne@69 121 Path eval(StringPtr pathText) &&;
jpayne@69 122 // Evaluates a traditional path relative to this one. `pathText` is parsed like `parse()` would,
jpayne@69 123 // except that:
jpayne@69 124 // - It can contain leading ".." components that traverse up the tree.
jpayne@69 125 // - It can have a leading '/' which completely replaces the current path.
jpayne@69 126 //
jpayne@69 127 // THE NAME OF THIS METHOD WAS CHOSEN TO INSPIRE FEAR.
jpayne@69 128 //
jpayne@69 129 // Instead of using `path.eval(str)`, always consider whether you really want
jpayne@69 130 // `path.append(Path::parse(str))`. The former is much riskier than the latter in terms of path
jpayne@69 131 // injection vulnerabilities.
jpayne@69 132
jpayne@69 133 PathPtr basename() const&;
jpayne@69 134 Path basename() &&;
jpayne@69 135 // Get the last component of the path. (Use `basename()[0]` to get just the string.)
jpayne@69 136
jpayne@69 137 PathPtr parent() const&;
jpayne@69 138 Path parent() &&;
jpayne@69 139 // Get the parent path.
jpayne@69 140
jpayne@69 141 String toString(bool absolute = false) const;
jpayne@69 142 // Converts the path to a traditional path string, appropriate to pass to a unix system call.
jpayne@69 143 // Never throws.
jpayne@69 144
jpayne@69 145 const String& operator[](size_t i) const&;
jpayne@69 146 String operator[](size_t i) &&;
jpayne@69 147 size_t size() const;
jpayne@69 148 const String* begin() const;
jpayne@69 149 const String* end() const;
jpayne@69 150 PathPtr slice(size_t start, size_t end) const&;
jpayne@69 151 Path slice(size_t start, size_t end) &&;
jpayne@69 152 // A Path can be accessed as an array of strings.
jpayne@69 153
jpayne@69 154 bool operator==(PathPtr other) const;
jpayne@69 155 bool operator!=(PathPtr other) const;
jpayne@69 156 bool operator< (PathPtr other) const;
jpayne@69 157 bool operator> (PathPtr other) const;
jpayne@69 158 bool operator<=(PathPtr other) const;
jpayne@69 159 bool operator>=(PathPtr other) const;
jpayne@69 160 // Compare path components lexically.
jpayne@69 161
jpayne@69 162 bool operator==(const Path& other) const;
jpayne@69 163 bool operator!=(const Path& other) const;
jpayne@69 164 bool operator< (const Path& other) const;
jpayne@69 165 bool operator> (const Path& other) const;
jpayne@69 166 bool operator<=(const Path& other) const;
jpayne@69 167 bool operator>=(const Path& other) const;
jpayne@69 168
jpayne@69 169 uint hashCode() const;
jpayne@69 170 // Can use in HashMap.
jpayne@69 171
jpayne@69 172 bool startsWith(PathPtr prefix) const;
jpayne@69 173 bool endsWith(PathPtr suffix) const;
jpayne@69 174 // Compare prefix / suffix.
jpayne@69 175
jpayne@69 176 Path evalWin32(StringPtr pathText) const&;
jpayne@69 177 Path evalWin32(StringPtr pathText) &&;
jpayne@69 178 // Evaluates a Win32-style path, as might be written by a user. Differences from `eval()`
jpayne@69 179 // include:
jpayne@69 180 //
jpayne@69 181 // - Backslashes can be used as path separators.
jpayne@69 182 // - Absolute paths begin with a drive letter followed by a colon. The drive letter, including
jpayne@69 183 // the colon, will become the first component of the path, e.g. "c:\foo" becomes {"c:", "foo"}.
jpayne@69 184 // - A network path like "\\host\share\path" is parsed as {"host", "share", "path"}.
jpayne@69 185
jpayne@69 186 Path evalNative(StringPtr pathText) const&;
jpayne@69 187 Path evalNative(StringPtr pathText) &&;
jpayne@69 188 // Alias for either eval() or evalWin32() depending on the target platform. Use this when you are
jpayne@69 189 // parsing a path provided by a user and you want the user to be able to use the "natural" format
jpayne@69 190 // for their platform.
jpayne@69 191
jpayne@69 192 String toWin32String(bool absolute = false) const;
jpayne@69 193 // Converts the path to a Win32 path string, as you might display to a user.
jpayne@69 194 //
jpayne@69 195 // This is meant for display. For making Win32 system calls, consider `toWin32Api()` instead.
jpayne@69 196 //
jpayne@69 197 // If `absolute` is true, the path is expected to be an absolute path, meaning the first
jpayne@69 198 // component is a drive letter, namespace, or network host name. These are converted to their
jpayne@69 199 // regular Win32 format -- i.e. this method does the reverse of `evalWin32()`.
jpayne@69 200 //
jpayne@69 201 // This throws if the path would have unexpected special meaning or is otherwise invalid on
jpayne@69 202 // Windows, such as if it contains backslashes (within a path component), colons, or special
jpayne@69 203 // names like "con".
jpayne@69 204
jpayne@69 205 String toNativeString(bool absolute = false) const;
jpayne@69 206 // Alias for either toString() or toWin32String() depending on the target platform. Use this when
jpayne@69 207 // you are formatting a path to display to a user and you want to present it in the "natural"
jpayne@69 208 // format for the user's platform.
jpayne@69 209
jpayne@69 210 Array<wchar_t> forWin32Api(bool absolute) const;
jpayne@69 211 // Like toWin32String, but additionally:
jpayne@69 212 // - Converts the path to UTF-16, with a NUL terminator included.
jpayne@69 213 // - For absolute paths, adds the "\\?\" prefix which opts into permitting paths longer than
jpayne@69 214 // MAX_PATH, and turns off relative path processing (which KJ paths already handle in userspace
jpayne@69 215 // anyway).
jpayne@69 216 //
jpayne@69 217 // This method is good to use when making a Win32 API call, e.g.:
jpayne@69 218 //
jpayne@69 219 // DeleteFileW(path.forWin32Api(true).begin());
jpayne@69 220
jpayne@69 221 static Path parseWin32Api(ArrayPtr<const wchar_t> text);
jpayne@69 222 // Parses an absolute path as returned by a Win32 API call like GetFinalPathNameByHandle() or
jpayne@69 223 // GetCurrentDirectory(). A "\\?\" prefix is optional but understood if present.
jpayne@69 224 //
jpayne@69 225 // Since such Win32 API calls generally return a length, this function inputs an array slice.
jpayne@69 226 // The slice should not include any NUL terminator.
jpayne@69 227
jpayne@69 228 private:
jpayne@69 229 Array<String> parts;
jpayne@69 230
jpayne@69 231 // TODO(perf): Consider unrolling one element from `parts`, so that a one-element path doesn't
jpayne@69 232 // require allocation of an array.
jpayne@69 233
jpayne@69 234 enum { ALREADY_CHECKED };
jpayne@69 235 Path(Array<String> parts, decltype(ALREADY_CHECKED));
jpayne@69 236
jpayne@69 237 friend class PathPtr;
jpayne@69 238
jpayne@69 239 static String stripNul(String input);
jpayne@69 240 static void validatePart(StringPtr part);
jpayne@69 241 static void evalPart(Vector<String>& parts, ArrayPtr<const char> part);
jpayne@69 242 static Path evalImpl(Vector<String>&& parts, StringPtr path);
jpayne@69 243 static Path evalWin32Impl(Vector<String>&& parts, StringPtr path, bool fromApi = false);
jpayne@69 244 static size_t countParts(StringPtr path);
jpayne@69 245 static size_t countPartsWin32(StringPtr path);
jpayne@69 246 static bool isWin32Drive(ArrayPtr<const char> part);
jpayne@69 247 static bool isNetbiosName(ArrayPtr<const char> part);
jpayne@69 248 static bool isWin32Special(StringPtr part);
jpayne@69 249 };
jpayne@69 250
jpayne@69 251 class PathPtr {
jpayne@69 252 // Points to a Path or a slice of a Path, but doesn't own it.
jpayne@69 253 //
jpayne@69 254 // PathPtr is to Path as ArrayPtr is to Array and StringPtr is to String.
jpayne@69 255
jpayne@69 256 public:
jpayne@69 257 PathPtr(decltype(nullptr));
jpayne@69 258 PathPtr(const Path& path);
jpayne@69 259
jpayne@69 260 Path clone();
jpayne@69 261 Path append(Path&& suffix) const;
jpayne@69 262 Path append(PathPtr suffix) const;
jpayne@69 263 Path append(StringPtr suffix) const;
jpayne@69 264 Path append(String&& suffix) const;
jpayne@69 265 Path eval(StringPtr pathText) const;
jpayne@69 266 PathPtr basename() const;
jpayne@69 267 PathPtr parent() const;
jpayne@69 268 String toString(bool absolute = false) const;
jpayne@69 269 const String& operator[](size_t i) const;
jpayne@69 270 size_t size() const;
jpayne@69 271 const String* begin() const;
jpayne@69 272 const String* end() const;
jpayne@69 273 PathPtr slice(size_t start, size_t end) const;
jpayne@69 274 bool operator==(PathPtr other) const;
jpayne@69 275 bool operator!=(PathPtr other) const;
jpayne@69 276 bool operator< (PathPtr other) const;
jpayne@69 277 bool operator> (PathPtr other) const;
jpayne@69 278 bool operator<=(PathPtr other) const;
jpayne@69 279 bool operator>=(PathPtr other) const;
jpayne@69 280 uint hashCode() const;
jpayne@69 281 bool startsWith(PathPtr prefix) const;
jpayne@69 282 bool endsWith(PathPtr suffix) const;
jpayne@69 283 Path evalWin32(StringPtr pathText) const;
jpayne@69 284 Path evalNative(StringPtr pathText) const;
jpayne@69 285 String toWin32String(bool absolute = false) const;
jpayne@69 286 String toNativeString(bool absolute = false) const;
jpayne@69 287 Array<wchar_t> forWin32Api(bool absolute) const;
jpayne@69 288 // Equivalent to the corresponding methods of `Path`.
jpayne@69 289
jpayne@69 290 private:
jpayne@69 291 ArrayPtr<const String> parts;
jpayne@69 292
jpayne@69 293 explicit PathPtr(ArrayPtr<const String> parts);
jpayne@69 294
jpayne@69 295 String toWin32StringImpl(bool absolute, bool forApi) const;
jpayne@69 296
jpayne@69 297 friend class Path;
jpayne@69 298 };
jpayne@69 299
jpayne@69 300 // =======================================================================================
jpayne@69 301 // The filesystem API
jpayne@69 302 //
jpayne@69 303 // This API is strictly synchronous because, unfortunately, there's no such thing as asynchronous
jpayne@69 304 // filesystem access in practice. The filesystem drivers on Linux are written to assume they can
jpayne@69 305 // block. The AIO API is only actually asynchronous for reading/writing the raw file blocks, but if
jpayne@69 306 // the filesystem needs to be involved (to allocate blocks, update metadata, etc.) that will block.
jpayne@69 307 // It's best to imagine that the filesystem is just another tier of memory that happens to be
jpayne@69 308 // slower than RAM (which is slower than L3 cache, which is slower than L2, which is slower than
jpayne@69 309 // L1). You can't do asynchronous RAM access so why asynchronous filesystem? The only way to
jpayne@69 310 // parallelize these is using threads.
jpayne@69 311 //
jpayne@69 312 // All KJ filesystem objects are thread-safe, and so all methods are marked "const" (even write
jpayne@69 313 // methods). Of course, if you concurrently write the same bytes of a file from multiple threads,
jpayne@69 314 // it's unspecified which write will "win".
jpayne@69 315
jpayne@69 316 class FsNode {
jpayne@69 317 // Base class for filesystem node types.
jpayne@69 318
jpayne@69 319 public:
jpayne@69 320 Own<const FsNode> clone() const;
jpayne@69 321 // Creates a new object of exactly the same type as this one, pointing at exactly the same
jpayne@69 322 // external object.
jpayne@69 323 //
jpayne@69 324 // Under the hood, this will call dup(), so the FD number will not be the same.
jpayne@69 325
jpayne@69 326 virtual Maybe<int> getFd() const { return nullptr; }
jpayne@69 327 // Get the underlying Unix file descriptor, if any. Returns nullptr if this object actually isn't
jpayne@69 328 // wrapping a file descriptor.
jpayne@69 329
jpayne@69 330 virtual Maybe<void*> getWin32Handle() const { return nullptr; }
jpayne@69 331 // Get the underlying Win32 HANDLE, if any. Returns nullptr if this object actually isn't
jpayne@69 332 // wrapping a handle.
jpayne@69 333
jpayne@69 334 enum class Type {
jpayne@69 335 FILE,
jpayne@69 336 DIRECTORY,
jpayne@69 337 SYMLINK,
jpayne@69 338 BLOCK_DEVICE,
jpayne@69 339 CHARACTER_DEVICE,
jpayne@69 340 NAMED_PIPE,
jpayne@69 341 SOCKET,
jpayne@69 342 OTHER,
jpayne@69 343 };
jpayne@69 344
jpayne@69 345 struct Metadata {
jpayne@69 346 Type type = Type::FILE;
jpayne@69 347
jpayne@69 348 uint64_t size = 0;
jpayne@69 349 // Logical size of the file.
jpayne@69 350
jpayne@69 351 uint64_t spaceUsed = 0;
jpayne@69 352 // Physical size of the file on disk. May be smaller for sparse files, or larger for
jpayne@69 353 // pre-allocated files.
jpayne@69 354
jpayne@69 355 Date lastModified = UNIX_EPOCH;
jpayne@69 356 // Last modification time of the file.
jpayne@69 357
jpayne@69 358 uint linkCount = 1;
jpayne@69 359 // Number of hard links pointing to this node.
jpayne@69 360
jpayne@69 361 uint64_t hashCode = 0;
jpayne@69 362 // Hint which can be used to determine if two FsNode instances point to the same underlying
jpayne@69 363 // file object. If two FsNodes report different hashCodes, then they are not the same object.
jpayne@69 364 // If they report the same hashCode, then they may or may not be the same object.
jpayne@69 365 //
jpayne@69 366 // The Unix filesystem implementation builds the hashCode based on st_dev and st_ino of
jpayne@69 367 // `struct stat`. However, note that some filesystems -- especially FUSE-based -- may not fill
jpayne@69 368 // in st_ino.
jpayne@69 369 //
jpayne@69 370 // The Windows filesystem implementation builds the hashCode based on dwVolumeSerialNumber and
jpayne@69 371 // dwFileIndex{Low,High} of the BY_HANDLE_FILE_INFORMATION structure. However, these are again
jpayne@69 372 // not guaranteed to be unique on all filesystems. In particular the documentation says that
jpayne@69 373 // ReFS uses 128-bit identifiers which can't be represented here, and again virtual filesystems
jpayne@69 374 // may often not report real identifiers.
jpayne@69 375 //
jpayne@69 376 // Of course, the process of hashing values into a single hash code can also cause collisions
jpayne@69 377 // even if the filesystem reports reliable information.
jpayne@69 378 //
jpayne@69 379 // Additionally note that this value is not reliable when returned by `lstat()`. You should
jpayne@69 380 // actually open the object, then call `stat()` on the opened object.
jpayne@69 381
jpayne@69 382 // Not currently included:
jpayne@69 383 // - Access control info: Differs wildly across platforms, and KJ prefers capabilities anyway.
jpayne@69 384 // - Other timestamps: Differs across platforms.
jpayne@69 385 // - Device number: If you care, you're probably doing platform-specific stuff anyway.
jpayne@69 386
jpayne@69 387 Metadata() = default;
jpayne@69 388 Metadata(Type type, uint64_t size, uint64_t spaceUsed, Date lastModified, uint linkCount,
jpayne@69 389 uint64_t hashCode)
jpayne@69 390 : type(type), size(size), spaceUsed(spaceUsed), lastModified(lastModified),
jpayne@69 391 linkCount(linkCount), hashCode(hashCode) {}
jpayne@69 392 // TODO(cleanup): This constructor is redundant in C++14, but needed in C++11.
jpayne@69 393 };
jpayne@69 394
jpayne@69 395 virtual Metadata stat() const = 0;
jpayne@69 396
jpayne@69 397 virtual void sync() const = 0;
jpayne@69 398 virtual void datasync() const = 0;
jpayne@69 399 // Maps to fsync() and fdatasync() system calls.
jpayne@69 400 //
jpayne@69 401 // Also, when creating or overwriting a file, the first call to sync() atomically links the file
jpayne@69 402 // into the filesystem (*after* syncing the data), so than incomplete data is never visible to
jpayne@69 403 // other processes. (In practice this works by writing into a temporary file and then rename()ing
jpayne@69 404 // it.)
jpayne@69 405
jpayne@69 406 protected:
jpayne@69 407 virtual Own<const FsNode> cloneFsNode() const = 0;
jpayne@69 408 // Implements clone(). Required to return an object with exactly the same type as this one.
jpayne@69 409 // Hence, every subclass must implement this.
jpayne@69 410 };
jpayne@69 411
jpayne@69 412 class ReadableFile: public FsNode {
jpayne@69 413 public:
jpayne@69 414 Own<const ReadableFile> clone() const;
jpayne@69 415
jpayne@69 416 String readAllText() const;
jpayne@69 417 // Read all text in the file and return as a big string.
jpayne@69 418
jpayne@69 419 Array<byte> readAllBytes() const;
jpayne@69 420 // Read all bytes in the file and return as a big byte array.
jpayne@69 421 //
jpayne@69 422 // This differs from mmap() in that the read is performed all at once. Future changes to the file
jpayne@69 423 // do not affect the returned copy. Consider using mmap() instead, particularly for large files.
jpayne@69 424
jpayne@69 425 virtual size_t read(uint64_t offset, ArrayPtr<byte> buffer) const = 0;
jpayne@69 426 // Fills `buffer` with data starting at `offset`. Returns the number of bytes actually read --
jpayne@69 427 // the only time this is less than `buffer.size()` is when EOF occurs mid-buffer.
jpayne@69 428
jpayne@69 429 virtual Array<const byte> mmap(uint64_t offset, uint64_t size) const = 0;
jpayne@69 430 // Maps the file to memory read-only. The returned array always has exactly the requested size.
jpayne@69 431 // Depending on the capabilities of the OS and filesystem, the mapping may or may not reflect
jpayne@69 432 // changes that happen to the file after mmap() returns.
jpayne@69 433 //
jpayne@69 434 // Multiple calls to mmap() on the same file may or may not return the same mapping (it is
jpayne@69 435 // immutable, so there's no possibility of interference).
jpayne@69 436 //
jpayne@69 437 // If the file cannot be mmap()ed, an implementation may choose to allocate a buffer on the heap,
jpayne@69 438 // read into it, and return that. This should only happen if a real mmap() is impossible.
jpayne@69 439 //
jpayne@69 440 // The returned array is always exactly the size requested. However, accessing bytes beyond the
jpayne@69 441 // current end of the file may raise SIGBUS, or may simply return zero.
jpayne@69 442
jpayne@69 443 virtual Array<byte> mmapPrivate(uint64_t offset, uint64_t size) const = 0;
jpayne@69 444 // Like mmap() but returns a view that the caller can modify. Modifications will not be written
jpayne@69 445 // to the underlying file. Every call to this method returns a unique mapping. Changes made to
jpayne@69 446 // the underlying file by other clients may or may not be reflected in the mapping -- in fact,
jpayne@69 447 // some changes may be reflected while others aren't, even within the same mapping.
jpayne@69 448 //
jpayne@69 449 // In practice this is often implemented using copy-on-write pages. When you first write to a
jpayne@69 450 // page, a copy is made. Hence, changes to the underlying file within that page stop being
jpayne@69 451 // reflected in the mapping.
jpayne@69 452 };
jpayne@69 453
jpayne@69 454 class AppendableFile: public FsNode, public OutputStream {
jpayne@69 455 public:
jpayne@69 456 Own<const AppendableFile> clone() const;
jpayne@69 457
jpayne@69 458 // All methods are inherited.
jpayne@69 459 };
jpayne@69 460
jpayne@69 461 class WritableFileMapping {
jpayne@69 462 public:
jpayne@69 463 virtual ArrayPtr<byte> get() const = 0;
jpayne@69 464 // Gets the mapped bytes. The returned array can be modified, and those changes may be written to
jpayne@69 465 // the underlying file, but there is no guarantee that they are written unless you subsequently
jpayne@69 466 // call changed().
jpayne@69 467
jpayne@69 468 virtual void changed(ArrayPtr<byte> slice) const = 0;
jpayne@69 469 // Notifies the implementation that the given bytes have changed. For some implementations this
jpayne@69 470 // may be a no-op while for others it may be necessary in order for the changes to be written
jpayne@69 471 // back at all.
jpayne@69 472 //
jpayne@69 473 // `slice` must be a slice of `bytes()`.
jpayne@69 474
jpayne@69 475 virtual void sync(ArrayPtr<byte> slice) const = 0;
jpayne@69 476 // Implies `changed()`, and then waits until the range has actually been written to disk before
jpayne@69 477 // returning.
jpayne@69 478 //
jpayne@69 479 // `slice` must be a slice of `bytes()`.
jpayne@69 480 //
jpayne@69 481 // On Windows, this calls FlushViewOfFile(). The documentation for this function implies that in
jpayne@69 482 // some circumstances, to fully sync to physical disk, you may need to call FlushFileBuffers() on
jpayne@69 483 // the file HANDLE as well. The documentation is not very clear on when and why this is needed.
jpayne@69 484 // If you believe your program needs this, you can accomplish it by calling `.sync()` on the File
jpayne@69 485 // object after calling `.sync()` on the WritableFileMapping.
jpayne@69 486 };
jpayne@69 487
jpayne@69 488 class File: public ReadableFile {
jpayne@69 489 public:
jpayne@69 490 Own<const File> clone() const;
jpayne@69 491
jpayne@69 492 void writeAll(ArrayPtr<const byte> bytes) const;
jpayne@69 493 void writeAll(StringPtr text) const;
jpayne@69 494 // Completely replace the file with the given bytes or text.
jpayne@69 495
jpayne@69 496 virtual void write(uint64_t offset, ArrayPtr<const byte> data) const = 0;
jpayne@69 497 // Write the given data starting at the given offset in the file.
jpayne@69 498
jpayne@69 499 virtual void zero(uint64_t offset, uint64_t size) const = 0;
jpayne@69 500 // Write zeros to the file, starting at `offset` and continuing for `size` bytes. If the platform
jpayne@69 501 // supports it, this will "punch a hole" in the file, such that blocks that are entirely zeros
jpayne@69 502 // do not take space on disk.
jpayne@69 503
jpayne@69 504 virtual void truncate(uint64_t size) const = 0;
jpayne@69 505 // Set the file end pointer to `size`. If `size` is less than the current size, data past the end
jpayne@69 506 // is truncated. If `size` is larger than the current size, zeros are added to the end of the
jpayne@69 507 // file. If the platform supports it, blocks containing all-zeros will not be stored to disk.
jpayne@69 508
jpayne@69 509 virtual Own<const WritableFileMapping> mmapWritable(uint64_t offset, uint64_t size) const = 0;
jpayne@69 510 // Like ReadableFile::mmap() but returns a mapping for which any changes will be immediately
jpayne@69 511 // visible in other mappings of the file on the same system and will eventually be written back
jpayne@69 512 // to the file.
jpayne@69 513
jpayne@69 514 virtual size_t copy(uint64_t offset, const ReadableFile& from, uint64_t fromOffset,
jpayne@69 515 uint64_t size) const;
jpayne@69 516 // Copies bytes from one file to another.
jpayne@69 517 //
jpayne@69 518 // Copies `size` bytes or to EOF, whichever comes first. Returns the number of bytes actually
jpayne@69 519 // copied. Hint: Pass kj::maxValue for `size` to always copy to EOF.
jpayne@69 520 //
jpayne@69 521 // The copy is not atomic. Concurrent writes may lead to garbage results.
jpayne@69 522 //
jpayne@69 523 // The default implementation performs a series of reads and writes. Subclasses can often provide
jpayne@69 524 // superior implementations that offload the work to the OS or even implement copy-on-write.
jpayne@69 525 };
jpayne@69 526
jpayne@69 527 class ReadableDirectory: public FsNode {
jpayne@69 528 // Read-only subset of `Directory`.
jpayne@69 529
jpayne@69 530 public:
jpayne@69 531 Own<const ReadableDirectory> clone() const;
jpayne@69 532
jpayne@69 533 virtual Array<String> listNames() const = 0;
jpayne@69 534 // List the contents of this directory. Does NOT include "." nor "..".
jpayne@69 535
jpayne@69 536 struct Entry {
jpayne@69 537 FsNode::Type type;
jpayne@69 538 String name;
jpayne@69 539
jpayne@69 540 inline bool operator< (const Entry& other) const { return name < other.name; }
jpayne@69 541 inline bool operator> (const Entry& other) const { return name > other.name; }
jpayne@69 542 inline bool operator<=(const Entry& other) const { return name <= other.name; }
jpayne@69 543 inline bool operator>=(const Entry& other) const { return name >= other.name; }
jpayne@69 544 // Convenience comparison operators to sort entries by name.
jpayne@69 545 };
jpayne@69 546
jpayne@69 547 virtual Array<Entry> listEntries() const = 0;
jpayne@69 548 // List the contents of the directory including the type of each file. On some platforms and
jpayne@69 549 // filesystems, this is just as fast as listNames(), but on others it may require stat()ing each
jpayne@69 550 // file.
jpayne@69 551
jpayne@69 552 virtual bool exists(PathPtr path) const = 0;
jpayne@69 553 // Does the specified path exist?
jpayne@69 554 //
jpayne@69 555 // If the path is a symlink, the symlink is followed and the return value indicates if the target
jpayne@69 556 // exists. If you want to know if the symlink exists, use lstat(). (This implies that listNames()
jpayne@69 557 // may return names for which exists() reports false.)
jpayne@69 558
jpayne@69 559 FsNode::Metadata lstat(PathPtr path) const;
jpayne@69 560 virtual Maybe<FsNode::Metadata> tryLstat(PathPtr path) const = 0;
jpayne@69 561 // Gets metadata about the path. If the path is a symlink, it is not followed -- the metadata
jpayne@69 562 // describes the symlink itself. `tryLstat()` returns null if the path doesn't exist.
jpayne@69 563
jpayne@69 564 Own<const ReadableFile> openFile(PathPtr path) const;
jpayne@69 565 virtual Maybe<Own<const ReadableFile>> tryOpenFile(PathPtr path) const = 0;
jpayne@69 566 // Open a file for reading.
jpayne@69 567 //
jpayne@69 568 // `tryOpenFile()` returns null if the path doesn't exist. Other errors still throw exceptions.
jpayne@69 569
jpayne@69 570 Own<const ReadableDirectory> openSubdir(PathPtr path) const;
jpayne@69 571 virtual Maybe<Own<const ReadableDirectory>> tryOpenSubdir(PathPtr path) const = 0;
jpayne@69 572 // Opens a subdirectory.
jpayne@69 573 //
jpayne@69 574 // `tryOpenSubdir()` returns null if the path doesn't exist. Other errors still throw exceptions.
jpayne@69 575
jpayne@69 576 String readlink(PathPtr path) const;
jpayne@69 577 virtual Maybe<String> tryReadlink(PathPtr path) const = 0;
jpayne@69 578 // If `path` is a symlink, reads and returns the link contents.
jpayne@69 579 //
jpayne@69 580 // Note that tryReadlink() differs subtly from tryOpen*(). For example, tryOpenFile() throws if
jpayne@69 581 // the path is not a file (e.g. if it's a directory); it only returns null if the path doesn't
jpayne@69 582 // exist at all. tryReadlink() returns null if either the path doesn't exist, or if it does exist
jpayne@69 583 // but isn't a symlink. This is because if it were to throw instead, then almost every real-world
jpayne@69 584 // use case of tryReadlink() would be forced to perform an lstat() first for the sole purpose of
jpayne@69 585 // checking if it is a link, wasting a syscall and a path traversal.
jpayne@69 586 //
jpayne@69 587 // See Directory::symlink() for warnings about symlinks.
jpayne@69 588 };
jpayne@69 589
jpayne@69 590 enum class WriteMode {
jpayne@69 591 // Mode for opening a file (or directory) for write.
jpayne@69 592 //
jpayne@69 593 // (To open a file or directory read-only, do not specify a mode.)
jpayne@69 594 //
jpayne@69 595 // WriteMode is a bitfield. Hence, it overloads the bitwise logic operators. To check if a
jpayne@69 596 // particular bit is set in a bitfield, use kj::has(), like:
jpayne@69 597 //
jpayne@69 598 // if (kj::has(mode, WriteMode::MUST_EXIST)) {
jpayne@69 599 // requireExists(path);
jpayne@69 600 // }
jpayne@69 601 //
jpayne@69 602 // (`if (mode & WriteMode::MUST_EXIST)` doesn't work because WriteMode is an enum class, which
jpayne@69 603 // cannot be converted to bool. Alas, C++ does not allow you to define a conversion operator
jpayne@69 604 // on an enum type, so we can't define a conversion to bool.)
jpayne@69 605
jpayne@69 606 // -----------------------------------------
jpayne@69 607 // Core flags
jpayne@69 608 //
jpayne@69 609 // At least one of CREATE or MODIFY must be specified. Optionally, the two flags can be combined
jpayne@69 610 // with a bitwise-OR.
jpayne@69 611
jpayne@69 612 CREATE = 1,
jpayne@69 613 // Create a new empty file.
jpayne@69 614 //
jpayne@69 615 // When not combined with MODIFY, if the file already exists (including as a broken symlink),
jpayne@69 616 // tryOpenFile() returns null (and openFile() throws).
jpayne@69 617 //
jpayne@69 618 // When combined with MODIFY, if the path already exists, it will be opened as if CREATE hadn't
jpayne@69 619 // been specified at all. If the path refers to a broken symlink, the file at the target of the
jpayne@69 620 // link will be created (if its parent directory exists).
jpayne@69 621
jpayne@69 622 MODIFY = 2,
jpayne@69 623 // Modify an existing file.
jpayne@69 624 //
jpayne@69 625 // When not combined with CREATE, if the file doesn't exist (including if it is a broken symlink),
jpayne@69 626 // tryOpenFile() returns null (and openFile() throws).
jpayne@69 627 //
jpayne@69 628 // When combined with CREATE, if the path doesn't exist, it will be created as if MODIFY hadn't
jpayne@69 629 // been specified at all. If the path refers to a broken symlink, the file at the target of the
jpayne@69 630 // link will be created (if its parent directory exists).
jpayne@69 631
jpayne@69 632 // -----------------------------------------
jpayne@69 633 // Additional flags
jpayne@69 634 //
jpayne@69 635 // Any number of these may be OR'd with the core flags.
jpayne@69 636
jpayne@69 637 CREATE_PARENT = 4,
jpayne@69 638 // Indicates that if the target node's parent directory doesn't exist, it should be created
jpayne@69 639 // automatically, along with its parent, and so on. This creation is NOT atomic.
jpayne@69 640 //
jpayne@69 641 // This bit only makes sense with CREATE or REPLACE.
jpayne@69 642
jpayne@69 643 EXECUTABLE = 8,
jpayne@69 644 // Mark this file executable, if this is a meaningful designation on the host platform.
jpayne@69 645
jpayne@69 646 PRIVATE = 16,
jpayne@69 647 // Indicates that this file is sensitive and should have permissions masked so that it is only
jpayne@69 648 // accessible by the current user.
jpayne@69 649 //
jpayne@69 650 // When this is not used, the platform's default access control settings are used. On Unix,
jpayne@69 651 // that usually means the umask is applied. On Windows, it means permissions are inherited from
jpayne@69 652 // the parent.
jpayne@69 653 };
jpayne@69 654
jpayne@69 655 inline constexpr WriteMode operator|(WriteMode a, WriteMode b) {
jpayne@69 656 return static_cast<WriteMode>(static_cast<uint>(a) | static_cast<uint>(b));
jpayne@69 657 }
jpayne@69 658 inline constexpr WriteMode operator&(WriteMode a, WriteMode b) {
jpayne@69 659 return static_cast<WriteMode>(static_cast<uint>(a) & static_cast<uint>(b));
jpayne@69 660 }
jpayne@69 661 inline constexpr WriteMode operator+(WriteMode a, WriteMode b) {
jpayne@69 662 return static_cast<WriteMode>(static_cast<uint>(a) | static_cast<uint>(b));
jpayne@69 663 }
jpayne@69 664 inline constexpr WriteMode operator-(WriteMode a, WriteMode b) {
jpayne@69 665 return static_cast<WriteMode>(static_cast<uint>(a) & ~static_cast<uint>(b));
jpayne@69 666 }
jpayne@69 667 template <typename T, typename = EnableIf<__is_enum(T)>>
jpayne@69 668 bool has(T haystack, T needle) {
jpayne@69 669 return (static_cast<__underlying_type(T)>(haystack) &
jpayne@69 670 static_cast<__underlying_type(T)>(needle)) ==
jpayne@69 671 static_cast<__underlying_type(T)>(needle);
jpayne@69 672 }
jpayne@69 673
jpayne@69 674 enum class TransferMode {
jpayne@69 675 // Specifies desired behavior for Directory::transfer().
jpayne@69 676
jpayne@69 677 MOVE,
jpayne@69 678 // The node is moved to the new location, i.e. the old location is deleted. If possible, this
jpayne@69 679 // move is performed without copying, otherwise it is performed as a copy followed by a delete.
jpayne@69 680
jpayne@69 681 LINK,
jpayne@69 682 // The new location becomes a synonym for the old location (a "hard link"). Filesystems have
jpayne@69 683 // varying support for this -- typically, it is not supported on directories.
jpayne@69 684
jpayne@69 685 COPY
jpayne@69 686 // The new location becomes a copy of the old.
jpayne@69 687 //
jpayne@69 688 // Some filesystems may implement this in terms of copy-on-write.
jpayne@69 689 //
jpayne@69 690 // If the filesystem supports sparse files, COPY takes sparseness into account -- it will punch
jpayne@69 691 // holes in the target file where holes exist in the source file.
jpayne@69 692 };
jpayne@69 693
jpayne@69 694 class Directory: public ReadableDirectory {
jpayne@69 695 // Refers to a specific directory on disk.
jpayne@69 696 //
jpayne@69 697 // A `Directory` object *only* provides access to children of the directory, not parents. That
jpayne@69 698 // is, you cannot open the file "..", nor jump to the root directory with "/".
jpayne@69 699 //
jpayne@69 700 // On OSs that support it, a `Directory` is backed by an open handle to the directory node. This
jpayne@69 701 // means:
jpayne@69 702 // - If the directory is renamed on-disk, the `Directory` object still points at it.
jpayne@69 703 // - Opening files in the directory only requires the OS to traverse the path from the directory
jpayne@69 704 // to the file; it doesn't have to re-traverse all the way from the filesystem root.
jpayne@69 705 //
jpayne@69 706 // On Windows, a `Directory` object holds a lock on the underlying directory such that it cannot
jpayne@69 707 // be renamed nor deleted while the object exists. This is necessary because Windows does not
jpayne@69 708 // fully support traversing paths relative to file handles (it does for some operations but not
jpayne@69 709 // all), so the KJ filesystem implementation is forced to remember the full path and needs to
jpayne@69 710 // ensure that the path is not invalidated. If, in the future, Windows fully supports
jpayne@69 711 // handle-relative paths, KJ may stop locking directories in this way, so do not rely on this
jpayne@69 712 // behavior.
jpayne@69 713
jpayne@69 714 public:
jpayne@69 715 Own<const Directory> clone() const;
jpayne@69 716
jpayne@69 717 template <typename T>
jpayne@69 718 class Replacer {
jpayne@69 719 // Implements an atomic replacement of a file or directory, allowing changes to be made to
jpayne@69 720 // storage in a way that avoids losing data in a power outage and prevents other processes
jpayne@69 721 // from observing content in an inconsistent state.
jpayne@69 722 //
jpayne@69 723 // `T` may be `File` or `Directory`. For readability, the text below describes replacing a
jpayne@69 724 // file, but the logic is the same for directories.
jpayne@69 725 //
jpayne@69 726 // When you call `Directory::replaceFile()`, a temporary file is created, but the specified
jpayne@69 727 // path is not yet touched. You may call `get()` to obtain the temporary file object, through
jpayne@69 728 // which you may initialize its content, knowing that no other process can see it yet. The file
jpayne@69 729 // is atomically moved to its final path when you call `commit()`. If you destroy the Replacer
jpayne@69 730 // without calling commit(), the temporary file is deleted.
jpayne@69 731 //
jpayne@69 732 // Note that most operating systems sadly do not support creating a truly unnamed temporary file
jpayne@69 733 // and then linking it in later. Moreover, the file cannot necessarily be created in the system
jpayne@69 734 // temporary directory because it might not be on the same filesystem as the target. Therefore,
jpayne@69 735 // the replacement file may initially be created in the same directory as its eventual target.
jpayne@69 736 // The implementation of Directory will choose a name that is unique and "hidden" according to
jpayne@69 737 // the conventions of the filesystem. Additionally, the implementation of Directory will avoid
jpayne@69 738 // returning these temporary files from its list*() methods, in order to avoid observable
jpayne@69 739 // inconsistencies across platforms.
jpayne@69 740 public:
jpayne@69 741 explicit Replacer(WriteMode mode);
jpayne@69 742
jpayne@69 743 virtual const T& get() = 0;
jpayne@69 744 // Gets the File or Directory representing the replacement data. Fill in this object before
jpayne@69 745 // calling commit().
jpayne@69 746
jpayne@69 747 void commit();
jpayne@69 748 virtual bool tryCommit() = 0;
jpayne@69 749 // Commit the replacement.
jpayne@69 750 //
jpayne@69 751 // `tryCommit()` may return false based on the CREATE/MODIFY bits passed as the WriteMode when
jpayne@69 752 // the replacement was initiated. (If CREATE but not MODIFY was used, tryCommit() returns
jpayne@69 753 // false to indicate that the target file already existed. If MODIFY but not CREATE was used,
jpayne@69 754 // tryCommit() returns false to indicate that the file didn't exist.)
jpayne@69 755 //
jpayne@69 756 // `commit()` is atomic, meaning that there is no point in time at which other processes
jpayne@69 757 // observing the file will see it in an intermediate state -- they will either see the old
jpayne@69 758 // content or the complete new content. This includes in the case of a power outage or machine
jpayne@69 759 // failure: on recovery, the file will either be in the old state or the new state, but not in
jpayne@69 760 // some intermediate state.
jpayne@69 761 //
jpayne@69 762 // It's important to note that a power failure *after commit() returns* can still revert the
jpayne@69 763 // file to its previous state. That is, `commit()` does NOT guarantee that, upon return, the
jpayne@69 764 // new content is durable. In order to guarantee this, you must call `sync()` on the immediate
jpayne@69 765 // parent directory of the replaced file.
jpayne@69 766 //
jpayne@69 767 // Note that, sadly, not all filesystems / platforms are capable of supporting all of the
jpayne@69 768 // guarantees documented above. In such cases, commit() will make a best-effort attempt to do
jpayne@69 769 // what it claims. Some examples of possible problems include:
jpayne@69 770 // - Any guarantees about durability through a power outage probably require a journaling
jpayne@69 771 // filesystem.
jpayne@69 772 // - Many platforms do not support atomically replacing a non-empty directory. Linux does as
jpayne@69 773 // of kernel 3.15 (via the renameat2() syscall using RENAME_EXCHANGE). Where not supported,
jpayne@69 774 // the old directory will be moved away just before the replacement is moved into place.
jpayne@69 775 // - Many platforms do not support atomically requiring the existence or non-existence of a
jpayne@69 776 // file before replacing it. In these cases, commit() may have to perform the check as a
jpayne@69 777 // separate step, with a small window for a race condition.
jpayne@69 778 // - Many platforms do not support "unlinking" a non-empty directory, meaning that a replaced
jpayne@69 779 // directory will need to be deconstructed by deleting all contents. If another process has
jpayne@69 780 // the directory open when it is replaced, that process will observe the contents
jpayne@69 781 // disappearing after the replacement (actually, a swap) has taken place. This differs from
jpayne@69 782 // files, where a process that has opened a file before it is replaced will continue see the
jpayne@69 783 // file's old content unchanged after the replacement.
jpayne@69 784 // - On Windows, there are multiple ways to replace one file with another in a single system
jpayne@69 785 // call, but none are documented as being atomic. KJ always uses `MoveFileEx()` with
jpayne@69 786 // MOVEFILE_REPLACE_EXISTING. While the alternative `ReplaceFile()` is attractive for many
jpayne@69 787 // reasons, it has the critical problem that it cannot be used when the source file has open
jpayne@69 788 // file handles, which is generally the case when using Replacer.
jpayne@69 789
jpayne@69 790 protected:
jpayne@69 791 const WriteMode mode;
jpayne@69 792 };
jpayne@69 793
jpayne@69 794 using ReadableDirectory::openFile;
jpayne@69 795 using ReadableDirectory::openSubdir;
jpayne@69 796 using ReadableDirectory::tryOpenFile;
jpayne@69 797 using ReadableDirectory::tryOpenSubdir;
jpayne@69 798
jpayne@69 799 Own<const File> openFile(PathPtr path, WriteMode mode) const;
jpayne@69 800 virtual Maybe<Own<const File>> tryOpenFile(PathPtr path, WriteMode mode) const = 0;
jpayne@69 801 // Open a file for writing.
jpayne@69 802 //
jpayne@69 803 // `tryOpenFile()` returns null if the path is required to exist but doesn't (MODIFY or REPLACE)
jpayne@69 804 // or if the path is required not to exist but does (CREATE or RACE). These are the only cases
jpayne@69 805 // where it returns null -- all other types of errors (like "access denied") throw exceptions.
jpayne@69 806
jpayne@69 807 virtual Own<Replacer<File>> replaceFile(PathPtr path, WriteMode mode) const = 0;
jpayne@69 808 // Construct a file which, when ready, will be atomically moved to `path`, replacing whatever
jpayne@69 809 // is there already. See `Replacer<T>` for detalis.
jpayne@69 810 //
jpayne@69 811 // The `CREATE` and `MODIFY` bits of `mode` are not enforced until commit time, hence
jpayne@69 812 // `replaceFile()` has no "try" variant.
jpayne@69 813
jpayne@69 814 virtual Own<const File> createTemporary() const = 0;
jpayne@69 815 // Create a temporary file backed by this directory's filesystem, but which isn't linked into
jpayne@69 816 // the directory tree. The file is deleted from disk when all references to it have been dropped.
jpayne@69 817
jpayne@69 818 Own<AppendableFile> appendFile(PathPtr path, WriteMode mode) const;
jpayne@69 819 virtual Maybe<Own<AppendableFile>> tryAppendFile(PathPtr path, WriteMode mode) const = 0;
jpayne@69 820 // Opens the file for appending only. Useful for log files.
jpayne@69 821 //
jpayne@69 822 // If the underlying filesystem supports it, writes to the file will always be appended even if
jpayne@69 823 // other writers are writing to the same file at the same time -- however, some implementations
jpayne@69 824 // may instead assume that no other process is changing the file size between writes.
jpayne@69 825
jpayne@69 826 Own<const Directory> openSubdir(PathPtr path, WriteMode mode) const;
jpayne@69 827 virtual Maybe<Own<const Directory>> tryOpenSubdir(PathPtr path, WriteMode mode) const = 0;
jpayne@69 828 // Opens a subdirectory for writing.
jpayne@69 829
jpayne@69 830 virtual Own<Replacer<Directory>> replaceSubdir(PathPtr path, WriteMode mode) const = 0;
jpayne@69 831 // Construct a directory which, when ready, will be atomically moved to `path`, replacing
jpayne@69 832 // whatever is there already. See `Replacer<T>` for detalis.
jpayne@69 833 //
jpayne@69 834 // The `CREATE` and `MODIFY` bits of `mode` are not enforced until commit time, hence
jpayne@69 835 // `replaceSubdir()` has no "try" variant.
jpayne@69 836
jpayne@69 837 void symlink(PathPtr linkpath, StringPtr content, WriteMode mode) const;
jpayne@69 838 virtual bool trySymlink(PathPtr linkpath, StringPtr content, WriteMode mode) const = 0;
jpayne@69 839 // Create a symlink. `content` is the raw text which will be written into the symlink node.
jpayne@69 840 // How this text is interpreted is entirely dependent on the filesystem. Note in particular that:
jpayne@69 841 // - Windows will require a path that uses backslashes as the separator.
jpayne@69 842 // - InMemoryDirectory does not support symlinks containing "..".
jpayne@69 843 //
jpayne@69 844 // Unfortunately under many implementations symlink() can be used to break out of the directory
jpayne@69 845 // by writing an absolute path or utilizing "..". Do not call this method with a value for
jpayne@69 846 // `target` that you don't trust.
jpayne@69 847 //
jpayne@69 848 // `mode` must be CREATE or REPLACE, not MODIFY. CREATE_PARENT is honored but EXECUTABLE and
jpayne@69 849 // PRIVATE have no effect. `trySymlink()` returns false in CREATE mode when the target already
jpayne@69 850 // exists.
jpayne@69 851
jpayne@69 852 void transfer(PathPtr toPath, WriteMode toMode,
jpayne@69 853 PathPtr fromPath, TransferMode mode) const;
jpayne@69 854 void transfer(PathPtr toPath, WriteMode toMode,
jpayne@69 855 const Directory& fromDirectory, PathPtr fromPath,
jpayne@69 856 TransferMode mode) const;
jpayne@69 857 virtual bool tryTransfer(PathPtr toPath, WriteMode toMode,
jpayne@69 858 const Directory& fromDirectory, PathPtr fromPath,
jpayne@69 859 TransferMode mode) const;
jpayne@69 860 virtual Maybe<bool> tryTransferTo(const Directory& toDirectory, PathPtr toPath, WriteMode toMode,
jpayne@69 861 PathPtr fromPath, TransferMode mode) const;
jpayne@69 862 // Move, link, or copy a file/directory tree from one location to another.
jpayne@69 863 //
jpayne@69 864 // Filesystems vary in what kinds of transfers are allowed, especially for TransferMode::LINK,
jpayne@69 865 // and whether TransferMode::MOVE is implemented as an actual move vs. copy+delete.
jpayne@69 866 //
jpayne@69 867 // tryTransfer() returns false if the source location didn't exist, or when `toMode` is CREATE
jpayne@69 868 // and the target already exists. The default implementation implements only TransferMode::COPY.
jpayne@69 869 //
jpayne@69 870 // tryTransferTo() exists to implement double-dispatch. It should be called as a fallback by
jpayne@69 871 // implementations of tryTransfer() in cases where the target directory would otherwise fail or
jpayne@69 872 // perform a pessimal transfer. The default implementation returns nullptr, which the caller
jpayne@69 873 // should interpret as: "I don't have any special optimizations; do the obvious thing."
jpayne@69 874 //
jpayne@69 875 // `toMode` controls how the target path is created. CREATE_PARENT is honored but EXECUTABLE and
jpayne@69 876 // PRIVATE have no effect.
jpayne@69 877
jpayne@69 878 void remove(PathPtr path) const;
jpayne@69 879 virtual bool tryRemove(PathPtr path) const = 0;
jpayne@69 880 // Deletes/unlinks the given path. If the path names a directory, it is recursively deleted.
jpayne@69 881 //
jpayne@69 882 // tryRemove() returns false in the specific case that the path doesn't exist. remove() would
jpayne@69 883 // throw in this case. In all other error cases (like "access denied"), tryRemove() still throws;
jpayne@69 884 // it is only "does not exist" that produces a false return.
jpayne@69 885 //
jpayne@69 886 // WARNING: The Windows implementation of recursive deletion is currently not safe to call from a
jpayne@69 887 // privileged process to delete directories writable by unprivileged users, due to a race
jpayne@69 888 // condition in which the user could trick the algorithm into following a symlink and deleting
jpayne@69 889 // everything at the destination. This race condition is not present in the Unix
jpayne@69 890 // implementation. Fixing it for Windows would require rewriting a lot of code to use different
jpayne@69 891 // APIs. If you're interested, see the TODO(security) in filesystem-disk-win32.c++.
jpayne@69 892
jpayne@69 893 // TODO(someday):
jpayne@69 894 // - Support sockets? There's no openat()-like interface for sockets, so it's hard to support
jpayne@69 895 // them currently. Also you'd probably want to use them with the async library.
jpayne@69 896 // - Support named pipes? Unclear if there's a use case that isn't better-served by sockets.
jpayne@69 897 // Then again, they can be openat()ed.
jpayne@69 898 // - Support watching for changes (inotify). Probably also requires the async library. Also
jpayne@69 899 // lacks openat()-like semantics.
jpayne@69 900 // - xattrs -- linux-specific
jpayne@69 901 // - chown/chmod/etc. -- unix-specific, ACLs, eww
jpayne@69 902 // - set timestamps -- only needed by archiving programs/
jpayne@69 903 // - advisory locks
jpayne@69 904 // - sendfile?
jpayne@69 905 // - fadvise and such
jpayne@69 906
jpayne@69 907 private:
jpayne@69 908 static void commitFailed(WriteMode mode);
jpayne@69 909 };
jpayne@69 910
jpayne@69 911 class Filesystem {
jpayne@69 912 public:
jpayne@69 913 virtual const Directory& getRoot() const = 0;
jpayne@69 914 // Get the filesystem's root directory, as of the time the Filesystem object was created.
jpayne@69 915
jpayne@69 916 virtual const Directory& getCurrent() const = 0;
jpayne@69 917 // Get the filesystem's current directory, as of the time the Filesystem object was created.
jpayne@69 918
jpayne@69 919 virtual PathPtr getCurrentPath() const = 0;
jpayne@69 920 // Get the path from the root to the current directory, as of the time the Filesystem object was
jpayne@69 921 // created. Note that because a `Directory` does not provide access to its parent, if you want to
jpayne@69 922 // follow `..` from the current directory, you must use `getCurrentPath().eval("..")` or
jpayne@69 923 // `getCurrentPath().parent()`.
jpayne@69 924 //
jpayne@69 925 // This function attempts to determine the path as it appeared in the user's shell before this
jpayne@69 926 // program was started. That means, if the user had `cd`ed into a symlink, the path through that
jpayne@69 927 // symlink is returned, *not* the canonical path.
jpayne@69 928 //
jpayne@69 929 // Because of this, there is an important difference between how the operating system interprets
jpayne@69 930 // "../foo" and what you get when you write `getCurrentPath().eval("../foo")`: The former
jpayne@69 931 // will interpret ".." relative to the directory's canonical path, whereas the latter will
jpayne@69 932 // interpret it relative to the path shown in the user's shell. In practice, the latter is
jpayne@69 933 // almost always what the user wants! But the former behavior is what almost all commands do
jpayne@69 934 // in practice, and it leads to confusion. KJ commands should implement the behavior the user
jpayne@69 935 // expects.
jpayne@69 936 };
jpayne@69 937
jpayne@69 938 // =======================================================================================
jpayne@69 939
jpayne@69 940 Own<File> newInMemoryFile(const Clock& clock);
jpayne@69 941 Own<Directory> newInMemoryDirectory(const Clock& clock);
jpayne@69 942 // Construct file and directory objects which reside in-memory.
jpayne@69 943 //
jpayne@69 944 // InMemoryFile has the following special properties:
jpayne@69 945 // - The backing store is not sparse and never gets smaller even if you truncate the file.
jpayne@69 946 // - While a non-private memory mapping exists, the backing store cannot get larger. Any operation
jpayne@69 947 // which would expand it will throw.
jpayne@69 948 //
jpayne@69 949 // InMemoryDirectory has the following special properties:
jpayne@69 950 // - Symlinks are processed using Path::parse(). This implies that a symlink cannot point to a
jpayne@69 951 // parent directory -- InMemoryDirectory does not know its parent.
jpayne@69 952 // - link() can link directory nodes in addition to files.
jpayne@69 953 // - link() and rename() accept any kind of Directory as `fromDirectory` -- it doesn't need to be
jpayne@69 954 // another InMemoryDirectory. However, for rename(), the from path must be a directory.
jpayne@69 955
jpayne@69 956 Own<AppendableFile> newFileAppender(Own<const File> inner);
jpayne@69 957 // Creates an AppendableFile by wrapping a File. Note that this implementation assumes it is the
jpayne@69 958 // only writer. A correct implementation should always append to the file even if other writes
jpayne@69 959 // are happening simultaneously, as is achieved with the O_APPEND flag to open(2), but that
jpayne@69 960 // behavior is not possible to emulate on top of `File`.
jpayne@69 961
jpayne@69 962 #if _WIN32
jpayne@69 963 typedef AutoCloseHandle OsFileHandle;
jpayne@69 964 #else
jpayne@69 965 typedef AutoCloseFd OsFileHandle;
jpayne@69 966 #endif
jpayne@69 967
jpayne@69 968 Own<ReadableFile> newDiskReadableFile(OsFileHandle fd);
jpayne@69 969 Own<AppendableFile> newDiskAppendableFile(OsFileHandle fd);
jpayne@69 970 Own<File> newDiskFile(OsFileHandle fd);
jpayne@69 971 Own<ReadableDirectory> newDiskReadableDirectory(OsFileHandle fd);
jpayne@69 972 Own<Directory> newDiskDirectory(OsFileHandle fd);
jpayne@69 973 // Wrap a file descriptor (or Windows HANDLE) as various filesystem types.
jpayne@69 974
jpayne@69 975 Own<Filesystem> newDiskFilesystem();
jpayne@69 976 // Get at implementation of `Filesystem` representing the real filesystem.
jpayne@69 977 //
jpayne@69 978 // DO NOT CALL THIS except at the top level of your program, e.g. in main(). Anywhere else, you
jpayne@69 979 // should instead have your caller pass in a Filesystem object, or a specific Directory object,
jpayne@69 980 // or whatever it is that your code needs. This ensures that your code supports dependency
jpayne@69 981 // injection, which makes it more reusable and testable.
jpayne@69 982 //
jpayne@69 983 // newDiskFilesystem() reads the current working directory at the time it is called. The returned
jpayne@69 984 // object is not affected by subsequent calls to chdir().
jpayne@69 985
jpayne@69 986 // =======================================================================================
jpayne@69 987 // inline implementation details
jpayne@69 988
jpayne@69 989 inline Path::Path(decltype(nullptr)): parts(nullptr) {}
jpayne@69 990 inline Path::Path(std::initializer_list<StringPtr> parts)
jpayne@69 991 : Path(arrayPtr(parts.begin(), parts.end())) {}
jpayne@69 992 inline Path::Path(Array<String> parts, decltype(ALREADY_CHECKED))
jpayne@69 993 : parts(kj::mv(parts)) {}
jpayne@69 994 inline Path Path::clone() const { return PathPtr(*this).clone(); }
jpayne@69 995 inline Path Path::append(Path&& suffix) const& { return PathPtr(*this).append(kj::mv(suffix)); }
jpayne@69 996 inline Path Path::append(PathPtr suffix) const& { return PathPtr(*this).append(suffix); }
jpayne@69 997 inline Path Path::append(StringPtr suffix) const& { return append(Path(suffix)); }
jpayne@69 998 inline Path Path::append(StringPtr suffix) && { return kj::mv(*this).append(Path(suffix)); }
jpayne@69 999 inline Path Path::append(String&& suffix) const& { return append(Path(kj::mv(suffix))); }
jpayne@69 1000 inline Path Path::append(String&& suffix) && { return kj::mv(*this).append(Path(kj::mv(suffix))); }
jpayne@69 1001 inline Path Path::eval(StringPtr pathText) const& { return PathPtr(*this).eval(pathText); }
jpayne@69 1002 inline PathPtr Path::basename() const& { return PathPtr(*this).basename(); }
jpayne@69 1003 inline PathPtr Path::parent() const& { return PathPtr(*this).parent(); }
jpayne@69 1004 inline const String& Path::operator[](size_t i) const& { return parts[i]; }
jpayne@69 1005 inline String Path::operator[](size_t i) && { return kj::mv(parts[i]); }
jpayne@69 1006 inline size_t Path::size() const { return parts.size(); }
jpayne@69 1007 inline const String* Path::begin() const { return parts.begin(); }
jpayne@69 1008 inline const String* Path::end() const { return parts.end(); }
jpayne@69 1009 inline PathPtr Path::slice(size_t start, size_t end) const& {
jpayne@69 1010 return PathPtr(*this).slice(start, end);
jpayne@69 1011 }
jpayne@69 1012 inline bool Path::operator==(PathPtr other) const { return PathPtr(*this) == other; }
jpayne@69 1013 inline bool Path::operator!=(PathPtr other) const { return PathPtr(*this) != other; }
jpayne@69 1014 inline bool Path::operator< (PathPtr other) const { return PathPtr(*this) < other; }
jpayne@69 1015 inline bool Path::operator> (PathPtr other) const { return PathPtr(*this) > other; }
jpayne@69 1016 inline bool Path::operator<=(PathPtr other) const { return PathPtr(*this) <= other; }
jpayne@69 1017 inline bool Path::operator>=(PathPtr other) const { return PathPtr(*this) >= other; }
jpayne@69 1018 inline bool Path::operator==(const Path& other) const { return PathPtr(*this) == PathPtr(other); }
jpayne@69 1019 inline bool Path::operator!=(const Path& other) const { return PathPtr(*this) != PathPtr(other); }
jpayne@69 1020 inline bool Path::operator< (const Path& other) const { return PathPtr(*this) < PathPtr(other); }
jpayne@69 1021 inline bool Path::operator> (const Path& other) const { return PathPtr(*this) > PathPtr(other); }
jpayne@69 1022 inline bool Path::operator<=(const Path& other) const { return PathPtr(*this) <= PathPtr(other); }
jpayne@69 1023 inline bool Path::operator>=(const Path& other) const { return PathPtr(*this) >= PathPtr(other); }
jpayne@69 1024 inline uint Path::hashCode() const { return kj::hashCode(parts); }
jpayne@69 1025 inline bool Path::startsWith(PathPtr prefix) const { return PathPtr(*this).startsWith(prefix); }
jpayne@69 1026 inline bool Path::endsWith (PathPtr suffix) const { return PathPtr(*this).endsWith (suffix); }
jpayne@69 1027 inline String Path::toString(bool absolute) const { return PathPtr(*this).toString(absolute); }
jpayne@69 1028 inline Path Path::evalWin32(StringPtr pathText) const& {
jpayne@69 1029 return PathPtr(*this).evalWin32(pathText);
jpayne@69 1030 }
jpayne@69 1031 inline String Path::toWin32String(bool absolute) const {
jpayne@69 1032 return PathPtr(*this).toWin32String(absolute);
jpayne@69 1033 }
jpayne@69 1034 inline Array<wchar_t> Path::forWin32Api(bool absolute) const {
jpayne@69 1035 return PathPtr(*this).forWin32Api(absolute);
jpayne@69 1036 }
jpayne@69 1037
jpayne@69 1038 inline PathPtr::PathPtr(decltype(nullptr)): parts(nullptr) {}
jpayne@69 1039 inline PathPtr::PathPtr(const Path& path): parts(path.parts) {}
jpayne@69 1040 inline PathPtr::PathPtr(ArrayPtr<const String> parts): parts(parts) {}
jpayne@69 1041 inline Path PathPtr::append(StringPtr suffix) const { return append(Path(suffix)); }
jpayne@69 1042 inline Path PathPtr::append(String&& suffix) const { return append(Path(kj::mv(suffix))); }
jpayne@69 1043 inline const String& PathPtr::operator[](size_t i) const { return parts[i]; }
jpayne@69 1044 inline size_t PathPtr::size() const { return parts.size(); }
jpayne@69 1045 inline const String* PathPtr::begin() const { return parts.begin(); }
jpayne@69 1046 inline const String* PathPtr::end() const { return parts.end(); }
jpayne@69 1047 inline PathPtr PathPtr::slice(size_t start, size_t end) const {
jpayne@69 1048 return PathPtr(parts.slice(start, end));
jpayne@69 1049 }
jpayne@69 1050 inline bool PathPtr::operator!=(PathPtr other) const { return !(*this == other); }
jpayne@69 1051 inline bool PathPtr::operator> (PathPtr other) const { return other < *this; }
jpayne@69 1052 inline bool PathPtr::operator<=(PathPtr other) const { return !(other < *this); }
jpayne@69 1053 inline bool PathPtr::operator>=(PathPtr other) const { return !(*this < other); }
jpayne@69 1054 inline uint PathPtr::hashCode() const { return kj::hashCode(parts); }
jpayne@69 1055 inline String PathPtr::toWin32String(bool absolute) const {
jpayne@69 1056 return toWin32StringImpl(absolute, false);
jpayne@69 1057 }
jpayne@69 1058
jpayne@69 1059 #if _WIN32
jpayne@69 1060 inline Path Path::evalNative(StringPtr pathText) const& {
jpayne@69 1061 return evalWin32(pathText);
jpayne@69 1062 }
jpayne@69 1063 inline Path Path::evalNative(StringPtr pathText) && {
jpayne@69 1064 return kj::mv(*this).evalWin32(pathText);
jpayne@69 1065 }
jpayne@69 1066 inline String Path::toNativeString(bool absolute) const {
jpayne@69 1067 return toWin32String(absolute);
jpayne@69 1068 }
jpayne@69 1069 inline Path PathPtr::evalNative(StringPtr pathText) const {
jpayne@69 1070 return evalWin32(pathText);
jpayne@69 1071 }
jpayne@69 1072 inline String PathPtr::toNativeString(bool absolute) const {
jpayne@69 1073 return toWin32String(absolute);
jpayne@69 1074 }
jpayne@69 1075 #else
jpayne@69 1076 inline Path Path::evalNative(StringPtr pathText) const& {
jpayne@69 1077 return eval(pathText);
jpayne@69 1078 }
jpayne@69 1079 inline Path Path::evalNative(StringPtr pathText) && {
jpayne@69 1080 return kj::mv(*this).eval(pathText);
jpayne@69 1081 }
jpayne@69 1082 inline String Path::toNativeString(bool absolute) const {
jpayne@69 1083 return toString(absolute);
jpayne@69 1084 }
jpayne@69 1085 inline Path PathPtr::evalNative(StringPtr pathText) const {
jpayne@69 1086 return eval(pathText);
jpayne@69 1087 }
jpayne@69 1088 inline String PathPtr::toNativeString(bool absolute) const {
jpayne@69 1089 return toString(absolute);
jpayne@69 1090 }
jpayne@69 1091 #endif // _WIN32, else
jpayne@69 1092
jpayne@69 1093 inline Own<const FsNode> FsNode::clone() const { return cloneFsNode(); }
jpayne@69 1094 inline Own<const ReadableFile> ReadableFile::clone() const {
jpayne@69 1095 return cloneFsNode().downcast<const ReadableFile>();
jpayne@69 1096 }
jpayne@69 1097 inline Own<const AppendableFile> AppendableFile::clone() const {
jpayne@69 1098 return cloneFsNode().downcast<const AppendableFile>();
jpayne@69 1099 }
jpayne@69 1100 inline Own<const File> File::clone() const { return cloneFsNode().downcast<const File>(); }
jpayne@69 1101 inline Own<const ReadableDirectory> ReadableDirectory::clone() const {
jpayne@69 1102 return cloneFsNode().downcast<const ReadableDirectory>();
jpayne@69 1103 }
jpayne@69 1104 inline Own<const Directory> Directory::clone() const {
jpayne@69 1105 return cloneFsNode().downcast<const Directory>();
jpayne@69 1106 }
jpayne@69 1107
jpayne@69 1108 inline void Directory::transfer(
jpayne@69 1109 PathPtr toPath, WriteMode toMode, PathPtr fromPath, TransferMode mode) const {
jpayne@69 1110 return transfer(toPath, toMode, *this, fromPath, mode);
jpayne@69 1111 }
jpayne@69 1112
jpayne@69 1113 template <typename T>
jpayne@69 1114 inline Directory::Replacer<T>::Replacer(WriteMode mode): mode(mode) {}
jpayne@69 1115
jpayne@69 1116 template <typename T>
jpayne@69 1117 void Directory::Replacer<T>::commit() {
jpayne@69 1118 if (!tryCommit()) commitFailed(mode);
jpayne@69 1119 }
jpayne@69 1120
jpayne@69 1121 } // namespace kj
jpayne@69 1122
jpayne@69 1123 KJ_END_HEADER