0% found this document useful (0 votes)
96 views

Zip Reader Function Library

This document contains code for reading zip files from disk at runtime in Unreal Engine using C++ and blueprints. It includes classes for reading zip files asynchronously in a worker thread and caching textures in memory.

Uploaded by

chairgraveyard
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as TXT, PDF, TXT or read online on Scribd
0% found this document useful (0 votes)
96 views

Zip Reader Function Library

This document contains code for reading zip files from disk at runtime in Unreal Engine using C++ and blueprints. It includes classes for reading zip files asynchronously in a worker thread and caching textures in memory.

Uploaded by

chairgraveyard
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as TXT, PDF, TXT or read online on Scribd
You are on page 1/ 6

// Copyright 1998-2013 Epic Games, Inc. All Rights Reserved.

#pragma once

#include "Engine/Texture2D.h"
#include "DDSLoader.h"
#include "TaskGraphInterfaces.h"
#include "Developer/ImageWrapper/Public/ImageWrapper.h"

#include "zlib.h"

#include "ZipReaderFunctionLibrary.generated.h"

// BP Library for reading zip files from disk at runtime.


//
// Written by n00854180t

#define JZ_BUFFER_SIZE 2048

class FZipReaderWorker : public FRunnable


{
/** Thread to run the worker FRunnable on */
FRunnableThread* Thread;

bool resize;
FString filename;

unsigned char tmpBuffer[JZ_BUFFER_SIZE];


bool mipDone;
bool decompressDone;
void* compressedBuffer;
void* tmpDecompressedBuffer;
void** outMipData;
int32 compressedSize;
int32 uncompressedSize;

unsigned char *bytes;


void* workingcompressedBuf;

long compressedLeft, uncompressedLeft;


z_stream strm;

int32 height, width;


int32* outWidth;
int32* outHeight;
TArray<uint8>* RawImgData;

IImageWrapperPtr ImageWrapper;

int32 x, y;

/** Stop this thread? Uses Thread Safe Counter */


FThreadSafeCounter StopTaskCounter;

int32 uncompress();

private:
int32 bytesRemaining;

public:

int32 TotalPrimesToFind;

//Done?
bool IsFinished() const
{
return bytesRemaining == 0 && x >= width && y >= height;
}

//~~~ Thread Core Functions ~~~

//Constructor / Destructor
FZipReaderWorker(FString Filename, void* CompressedBuffer, void** OutMipData,
int32 CompressedSize, int32 UncompressedSize, int32* OutWidth, int32* OutHeight,
IImageWrapperPtr ImgWrapper, bool Resize);
virtual ~FZipReaderWorker();

// Begin FRunnable interface.


virtual bool Init();
virtual uint32 Run();
virtual void Stop();
// End FRunnable interface

/** Makes sure this thread has stopped properly */


void EnsureCompletion();

//~~~ Starting and Stopping Thread ~~~


/*
Start the thread and the worker from static (easy access)!
This code ensures only 1 Prime Number thread will be able to run at a time.
This function returns a handle to the newly started instance.
*/
static FZipReaderWorker* InitWorker(FString filename, void* compressedBuffer,
void** outMipData, int32 compressedSize, int32 uncompressedSize, int32* outWidth,
int32* outHeight, IImageWrapperPtr imgWrapper, bool resize);

/** Shuts down the thread. Static so it can easily be called from outside the
thread context */
void Shutdown();

bool IsThreadFinished();

};

struct CompressedChunkInfo
{
// Default constructor, zero initializing all members.
CompressedChunkInfo()
{
UncompressedSize = 0;
CompressedSize = 0;
CompressedOffset = 0;
}

// Uncompressed size in bytes.


int32 UncompressedSize;
/// Offset in compressed file.
int32 CompressedOffset;
/// Compressed size in bytes.
int32 CompressedSize;

// Name of this compressed file, if applicable.


FString ChunkFileName;
};

struct CachedTextureInfo
{
CachedTextureInfo()
{
PageNumber = 0;
Width = 0;
Height = 0;
TmpMipData = NULL;
Texture = NULL;
}

int32 PageNumber;
void* TmpMipData;
int32 Width;
int32 Height;
TWeakObjectPtr<UTexture2D> Texture;
};

struct ZipFile
{
// Default constructor, zero initializing all members.
ZipFile()
{
CompressedData = NULL;
RequestId = -1;
IsCompressedReadComplete = false;

MainPagesLoaded = 0;
SecondaryPagesLoaded = 0;

FileSize = 0;
}

// Zip file name.


FString FileName;

// Compressed zip data.


void* CompressedData;

// Size of zip.
int32 FileSize;

// Has the compressed read been requested.


uint32 RequestId;

// Is the async read of compressed zip data complete?


bool IsCompressedReadComplete;

// Info about each file in the zip.


TArray<CompressedChunkInfo> FileChunks;

// Main page cache - limited number of full-resolution pages kept in memory.


TArray<CachedTextureInfo> MainTextureCache;

// Secondary page cache - low resolution version of all pages, to show if a


high res is not available, while loading.
TArray<CachedTextureInfo> SecondaryTextureCache;

// A list of pages requested from this zip that haven't been dispatched for
load.
TArray<int32> PendingPageRequests;

// A list of page requests with jobs that have been started already.
TArray<int32> ActivePageRequests;

// Number of hi res pages loaded.


int32 MainPagesLoaded;

// Number of secondary low res pages loaded.


int32 SecondaryPagesLoaded;
};

typedef struct {
#pragma pack(push,1) // Supposedly, this is supported on GCC as of certain
versions, definitely 4.0 and greater. *Important for porting - make sure that
something equivalent is done for your compiler, otherwise archives will fail to
load due to wrong offsets in reading/seeking.
uint32 signature;
uint16 versionNeededToExtract; // unsupported
uint16 generalPurposeBitFlag; // unsupported
uint16 compressionMethod;
uint16 lastModFileTime;
uint16 lastModFileDate;
uint32 crc32;
uint32 compressedSize;
uint32 uncompressedSize;
uint16 fileNameLength;
uint16 extraFieldLength; // unsupported
#pragma pack(pop)
} JZLocalFileHeader;

typedef struct {
#pragma pack(push,1)
uint32 signature;
uint16 versionMadeBy; // unsupported
uint16 versionNeededToExtract; // unsupported
uint16 generalPurposeBitFlag; // unsupported
uint16 compressionMethod;
uint16 lastModFileTime;
uint16 lastModFileDate;
uint32 crc32;
uint32 compressedSize;
uint32 uncompressedSize;
uint16 fileNameLength;
uint16 extraFieldLength; // unsupported
uint16 fileCommentLength; // unsupported
uint16 diskNumberStart; // unsupported
uint16 internalFileAttributes; // unsupported
uint32 externalFileAttributes; // unsupported
uint32 relativeOffsetOflocalHeader;
#pragma pack(pop)
} JZGlobalFileHeader;

typedef struct {
#pragma pack(push,1)
uint16 compressionMethod;
uint16 lastModFileTime;
uint16 lastModFileDate;
uint32 crc32;
uint32 compressedSize;
uint32 uncompressedSize;
uint32 offset;
#pragma pack(pop)
} JZFileHeader;

typedef struct {
#pragma pack(push,1)
uint32 signature; // 0x06054b50
uint16 diskNumber; // unsupported
uint16 centralDirectoryDiskNumber; // unsupported
uint16 numEntriesThisDisk; // unsupported
uint16 numEntries;
uint32 centralDirectorySize;
uint32 centralDirectoryOffset;
uint16 zipCommentLength;
#pragma pack(pop)
// Followed by .ZIP file comment (variable size)
} JZEndRecord;

UCLASS()
class ZIPREADER_API UZipReaderFunctionLibrary : public UBlueprintFunctionLibrary
{
GENERATED_UCLASS_BODY()

//

UFUNCTION(BlueprintCallable, Category = "ZipReader")


static int32 GetFileCount(const FString& FilePath);

UFUNCTION(BlueprintCallable, Category = "ZipReader")


static class UTexture2D* CheckCachedTexture(int32 TextureIndex);

UFUNCTION(BlueprintCallable, Category = "ZipReader")


static void CleanUpCache();

//static void CheckAllThreadsDone();

//static int32 CollectSimpleZipInfo(IFileHandle *zip, JZEndRecord *endRecord,


SimpleZipData* outZipData);
static void CheckAllFilesLoaded();
static bool TasksAreComplete();
static class UTexture2D* GetTextureFromMipData(FString FileName, void*
SrcData, int32 Width, int32 Height, int32 Size);
UFUNCTION(BlueprintCallable, Category = "ZipReader")
static void QueueTextureLoad(const FString& ZipFileName, int32 TextureIndex);

UFUNCTION(BlueprintCallable, Category = "ZipReader")


static void QueueAllTextures(const FString& ZipFileName, int32 TextureCount);

private:

static TArray<FZipReaderWorker*> ReaderWorkers;

static TArray<UTexture2D*> CachedTextures;


//static SimpleZipData* CachedZipData;

static FTimerDelegate FinishedLoadingDelegate;


static FString CacheFileName;

static ZipFile* GetOrAddCachedZip(const FString& ZipFileName);

static void StartThreadCheckerTimer();

static class IFileHandle* OpenFromFileName(const FString& FileName);


static int32 CollectSimpleZipInfo(ZipFile* zip, JZEndRecord *endRecord, int32
offset);

static bool AddZipReadRequest(ZipFile* Zip, int32 ChunkIndex);

static TArray<ZipFile*> ZipFileCache;

// Thanks to @Ehamloptiran for the base code for loading a texture:


// https://docs.google.com/file/d/0BzqpBc4MrAMXLXN1ZGNzUEJZc1k/edit
};

// Read ZIP file end record. Will move within file.


int jzReadEndRecord(IFileHandle *ZipFile, JZEndRecord *endRecord);

// Read local ZIP file header. Silent on errors so optimistic reading possible.
int jzReadLocalFileHeader(IFileHandle *zip, JZFileHeader *header, char *filename,
int len);

// Read ZIP file end record. Will move within file.


int jzReadEndRecordFromMemory(void* zipData, JZEndRecord *endRecord, int32
currentOffset, int32 zipFileSize);

// Read local ZIP file header. Silent on errors so optimistic reading possible.
int jzReadLocalFileHeaderFromMemory(void* zipData, JZFileHeader *header, char
*filename, int len, int32 currentOffset);

You might also like