let FileReader::Read return an opaque buffer instead of std::vector.

This can later allow returning a pointer to a static buffer from a cache without creating copies.

src/common/audio/music/i_music.cpp

@@ -337,7 +337,7 @@ static ZMusic_MidiSource GetMIDISource(const char *fn)
 	}
 
 	auto data = wlump.Read();
-	auto source = ZMusic_CreateMIDISource(data.data(), data.size(), type);
+	auto source = ZMusic_CreateMIDISource(data.bytes(), data.size(), type);
 
 	if (source == nullptr)
 	{

src/common/cutscenes/movieplayer.cpp

@@ -153,7 +153,7 @@ class AnmPlayer : public MoviePlayer
 {
 	// This doesn't need its own class type
 	anim_t anim;
-	std::vector<uint8_t> buffer;
+	FileSys::ResourceData buffer;
 	int numframes = 0;
 	int curframe = 1;
 	int frametime = 0;
@@ -173,7 +173,7 @@ class AnmPlayer : public MoviePlayer
 		buffer = fr.ReadPadded(1);
 		fr.Close();
 
-		if (ANIM_LoadAnim(&anim, buffer.data(), buffer.size() - 1) < 0)
+		if (ANIM_LoadAnim(&anim, buffer.bytes(), buffer.size() - 1) < 0)
 		{
 			return;
 		}

src/common/filesystem/include/fs_files.h

@@ -90,6 +90,84 @@ enum
 
 class FileReader;
 
+// an opaque memory buffer to the file's content. Can either own the memory or just point to an external buffer.
+class ResourceData
+{
+	void* memory;
+	size_t length;
+	bool owned;
+
+public:
+	using value_type = uint8_t;
+	ResourceData() { memory = nullptr; length = 0; owned = true; }
+	const void* data() const { return memory; }
+	size_t size() const { return length; }
+	const char* string() const { return (const char*)memory; }
+	const uint8_t* bytes() const { return (const uint8_t*)memory; }
+
+	ResourceData& operator = (const ResourceData& copy)
+	{
+		if (owned && memory) free(memory);
+		length = copy.length;
+		owned = copy.owned;
+		if (owned)
+		{
+			memory = malloc(length);
+			memcpy(memory, copy.memory, length);
+		}
+		else memory = copy.memory;
+		return *this;
+	}
+
+	ResourceData& operator = (ResourceData&& copy) noexcept
+	{
+		if (owned && memory) free(memory);
+		length = copy.length;
+		owned = copy.owned;
+		memory = copy.memory;
+		copy.memory = nullptr;
+		copy.length = 0;
+		copy.owned = true;
+		return *this;
+	}
+
+	ResourceData(const ResourceData& copy)
+	{
+		memory = nullptr;
+		*this = copy;
+	}
+
+	~ResourceData()
+	{
+		if (owned && memory) free(memory);
+	}
+
+	void* allocate(size_t len)
+	{
+		if (!owned) memory = nullptr;
+		length = len;
+		owned = true;
+		memory = realloc(memory, length);
+		return memory;
+	}
+
+	void set(const void* mem, size_t len)
+	{
+		memory = (void*)mem;
+		length = len;
+		owned = false;
+	}
+
+	void clear()
+	{
+		if (owned && memory) free(memory);
+		memory = nullptr;
+		length = 0;
+		owned = true;
+	}
+
+};
+
 class FileReaderInterface
 {
 public:
@@ -171,10 +249,12 @@ class FileReader
 		mReader = nullptr;
 	}
 
-	bool OpenFile(const char *filename, Size start = 0, Size length = -1);
+	bool OpenFile(const char *filename, Size start = 0, Size length = -1, bool buffered = false);
 	bool OpenFilePart(FileReader &parent, Size start, Size length);
 	bool OpenMemory(const void *mem, Size length);	// read directly from the buffer
 	bool OpenMemoryArray(const void *mem, Size length);	// read from a copy of the buffer.
+	bool OpenMemoryArray(std::vector<uint8_t>& data);	// take the given array
+	bool OpenMemoryArray(ResourceData& data);	// take the given array
 	bool OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter);	// read contents to a buffer and return a reader to it
 	bool OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions = false);	// creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used.
 
@@ -193,33 +273,34 @@ class FileReader
 		return mReader->Read(buffer, len);
 	}
 
-	std::vector<uint8_t> Read(size_t len)
+	ResourceData Read(size_t len)
 	{
-		std::vector<uint8_t> buffer(len);
+		ResourceData buffer;
 		if (len > 0)
 		{
-			Size length = mReader->Read(&buffer[0], len);
-			buffer.resize((size_t)length);
+			Size length = mReader->Read(buffer.allocate(len), len);
+			if (length < len) buffer.allocate(length);
 		}
 		return buffer;
 	}
 
-	std::vector<uint8_t> Read()
+	ResourceData Read()
 	{
 		return Read(GetLength());
 	}
 
-	std::vector<uint8_t> ReadPadded(size_t padding)
+	ResourceData ReadPadded(size_t padding)
 	{
 		auto len = GetLength();
-		std::vector<uint8_t> buffer(len + padding);
+		ResourceData buffer;
+
 		if (len > 0)
 		{
-			Size length = mReader->Read(&buffer[0], len);
+			auto p = (char*)buffer.allocate(len + padding);
+			Size length = mReader->Read(p, len);
 			if (length < len) buffer.clear();
-			else memset(buffer.data() + len, 0, padding);
+			else memset(p + len, 0, padding);
 		}
-		else buffer[0] = 0;
 		return buffer;
 	}
 
@@ -353,13 +434,13 @@ class FileWriter
 class BufferWriter : public FileWriter
 {
 protected:
-	TArray<unsigned char> mBuffer;
+	std::vector<unsigned char> mBuffer;
 public:
 
 	BufferWriter() {}
 	virtual size_t Write(const void *buffer, size_t len) override;
-	TArray<unsigned char> *GetBuffer() { return &mBuffer; }
-	TArray<unsigned char>&& TakeBuffer() { return std::move(mBuffer); }
+	std::vector<unsigned char> *GetBuffer() { return &mBuffer; }
+	std::vector<unsigned char>&& TakeBuffer() { return std::move(mBuffer); }
 };
 
 }

src/common/filesystem/include/resourcefile.h

@@ -88,8 +88,8 @@ enum ELumpFlags
 // This holds a compresed Zip entry with all needed info to decompress it.
 struct FCompressedBuffer
 {
-	unsigned mSize;
-	unsigned mCompressedSize;
+	size_t mSize;
+	size_t mCompressedSize;
 	int mMethod;
 	int mZipFlags;
 	unsigned mCRC32;

src/common/filesystem/source/files.cpp

@@ -34,6 +34,7 @@
 */
 
 #include <string>
+#include <memory>
 #include "files_internal.h"
 
 namespace FileSys {
@@ -285,7 +286,7 @@ ptrdiff_t MemoryReader::Seek(ptrdiff_t offset, int origin)
 
 ptrdiff_t MemoryReader::Read(void *buffer, ptrdiff_t len)
 {
-	if (len>Length - FilePos) len = Length - FilePos;
+	if (len > Length - FilePos) len = Length - FilePos;
 	if (len<0) len = 0;
 	memcpy(buffer, bufptr + FilePos, len);
 	FilePos += len;
@@ -322,40 +323,37 @@ char *MemoryReader::Gets(char *strbuf, ptrdiff_t len)
 	return strbuf;
 }
 
-//==========================================================================
-//
-// MemoryArrayReader
-//
-// reads data from an array of memory
-//
-//==========================================================================
-
-class MemoryArrayReader : public MemoryReader
+int BufferingReader::FillBuffer(size_t newpos)
 {
-	std::vector<uint8_t> buf;
-
-public:
-	MemoryArrayReader(const char *buffer, ptrdiff_t length)
+	if (newpos > Length) newpos = Length;
+	if (newpos < bufferpos) return 0;
+	auto read = baseReader->Read(&buf[bufferpos], newpos - bufferpos);
+	bufferpos += read;
+	if (bufferpos == Length)
 	{
-		if (length > 0)
-		{
-			buf.resize(length);
-			memcpy(&buf[0], buffer, length);
-		}
-		UpdateBuffer();
+		// we have read the entire file, so delete our data provider.
+		baseReader.reset();
 	}
+	return read == newpos - bufferpos ? 0 : -1;
+}
 
-	std::vector<uint8_t> &GetArray() { return buf; }
-
-	void UpdateBuffer() 
-	{ 
-		bufptr = (const char*)buf.data();
-		FilePos = 0;
-		Length = buf.size();
-	}
-};
+ptrdiff_t BufferingReader::Seek(ptrdiff_t offset, int origin)
+{
+	if (-1 == MemoryReader::Seek(offset, origin)) return -1;
+	return FillBuffer(FilePos);
+}
 
+ptrdiff_t BufferingReader::Read(void* buffer, ptrdiff_t len)
+{
+	if (FillBuffer(FilePos + len) < 0) return 0;
+	return MemoryReader::Read(buffer, len);
+}
 
+char* BufferingReader::Gets(char* strbuf, ptrdiff_t len)
+{
+	if (FillBuffer(FilePos + len) < 0) return nullptr;
+	return MemoryReader::Gets(strbuf, len);
+}
 
 //==========================================================================
 //
@@ -365,7 +363,7 @@ class MemoryArrayReader : public MemoryReader
 //
 //==========================================================================
 
-bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileReader::Size length)
+bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileReader::Size length, bool buffered)
 {
 	auto reader = new StdFileReader;
 	if (!reader->Open(filename, start, length))
@@ -374,7 +372,8 @@ bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileRead
 		return false;
 	}
 	Close();
-	mReader = reader;
+	if (buffered) mReader = new BufferingReader(reader);
+	else mReader = reader;
 	return true;
 }
 
@@ -396,13 +395,27 @@ bool FileReader::OpenMemory(const void *mem, FileReader::Size length)
 bool FileReader::OpenMemoryArray(const void *mem, FileReader::Size length)
 {
 	Close();
-	mReader = new MemoryArrayReader((const char *)mem, length);
+	mReader = new MemoryArrayReader<std::vector<uint8_t>>((const char *)mem, length);
+	return true;
+}
+
+bool FileReader::OpenMemoryArray(std::vector<uint8_t>& data)
+{
+	Close();
+	if (data.size() > 0) mReader = new MemoryArrayReader<std::vector<uint8_t>>(data);
+	return true;
+}
+
+bool FileReader::OpenMemoryArray(ResourceData& data)
+{
+	Close();
+	if (data.size() > 0) mReader = new MemoryArrayReader<ResourceData>(data);
 	return true;
 }
 
 bool FileReader::OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter)
 {
-	auto reader = new MemoryArrayReader(nullptr, 0);
+	auto reader = new MemoryArrayReader<std::vector<uint8_t>>(nullptr, 0);
 	if (getter(reader->GetArray()))
 	{
 		Close();
@@ -494,7 +507,8 @@ size_t FileWriter::Printf(const char *fmt, ...)
 
 size_t BufferWriter::Write(const void *buffer, size_t len)
 {
-	unsigned int ofs = mBuffer.Reserve((unsigned)len);
+	size_t ofs = mBuffer.size();
+	mBuffer.resize(ofs + len);
 	memcpy(&mBuffer[ofs], buffer, len);
 	return len;
 }