Removal of old caching

This commit is contained in:
René Linder 2023-03-29 10:41:36 +02:00
parent 72fc4436c9
commit 6a57a9f393
2 changed files with 352 additions and 61 deletions

129
inc/lcmf_main.pbi Normal file
View file

@ -0,0 +1,129 @@
;********************************
;*
;* lcmf_main.pbi
;*
;* (c) Cache (m) Memory and (f) File
;*
;* LiHaSo Webserver Modul In Memory File Cache
;*
;* It Could be configured Globaly and by Virtualhost
;*
DeclareModule lcmf
#Cache_Server_Single = 0
#Cache_Server_Multi = 1
EnumerationBinary lcmf_init
#Mem_Cache
#File_Cache
EndEnumeration
EnumerationBinary lcmf_object_state
#NA ;Not available
#InProgress ;Copy is in Progress
#Cached ;Object is available
#Cleaning ;Object is on the way to cleaned out
EndEnumeration
EnumerationBinary lcmf_params_state
;#UCP = Universal Cache Params (File and Mem)
#UCP_Max_Size_Total ;Total Size of Cache
#UCP_Max_Size_per_Entry ;Max Size of a File or Memoryblock
;#FAC = File Access Config
#FAC_MC ;Minimum Access Times until a File is Cached (Generic)
#FAC_RT ;Max Time a File could be in Cache (Generic)
#FAC_LA_RT ;How long a File could stay in Cache sinde last Access (Generic)
;(If a File has regulary access For ex. every 10 min. And this lifetime is 3600min. it will be killed after FAC_RT is reached)
#FAC_WL_Add ;Whitelist add Generic (Ex. :
;".html" = All html files
;"file.html" Expliciz Filenames
;"/path/" Including all Files and Subdirectorys It is added to Valid_Source
;"/path" only Files in Directory
;"'/path/','.rpm'" Combines a path and a Specific file extension
#FAC_WL_Add_Ext ;Whitelist which add with special MC/RT/LA_RT Params
#FAC_Valid_Source ;From where the files could get
#FAC_Cache_Dir ;Directory where the Cached Files could be written
;Some Function for Set_State who do a active Work
#FAC_Check_Cache ;Checks if all Files are the Same as Source
#UCP_Flush_Object ;Removes a Object from Cache
EndEnumeration
Declare.s Init(Server_Type.i = #Cache_Server_Multi) ;Create a Main Cache_Thread Management Server
Declare.s Init_cache(Cache_Type.i = #Mem_Cache) ;Create a Cache_Thread gives back the UUID
Declare.i Set_Params(Server_ID.s, Parameters.s) ;Set Cache Parameters in pre run state
Declare.i Use_New_Params(Server_ID.s) ;Reinit the Cache Thread
Declare.i Run_Cache(Server_ID.s) ;Start a initialised Cache Init()->InitCache()->Set_Params()->Run_Cache()
Declare.i Kill_Cache(Server_ID.s) ;Kills a Cache Server Thread
Declare.i Set_State(Server_ID.s, Parameters.s) ;Force to do something in Cache Thread (Flush a Element, Cleanup the Filecache-IO Intesitive, Check for Updates on Files an co.)
Declare.s Get_State(Server_ID.s, Parameters.s) ;Get Cache Server Thread State (For ex. current elements cachced, sizes and co.)
Declare.s Put_Mem(Server_ID.s, Memblock_ToSave.i, MC.i=-1, RT.i=-1, LA_RT.i=-1) ;Push a Memory Block to cache (Ex. some pre executed scripts or pictures and so on.)
Declare.s Get_Mem(Server_ID.s, Memblock_ID.s, MC.i=-1, RT.i=-1, LA_RT.i=-1) ;Get That Memoryblock from cache who was Previously Stored
Declare.s Get_File(Server_ID.s, Full_File_Name.s, Force_Disk.i = #True) ;Get a File (Cached or uncached) could be used everytime (ServerUUID empty everytime uncached or Flag uncached is set)
EndDeclareModule
Module lcmf
IncludeFile "lhs_uuid.pbi"
Structure lcmf_mem_objects
MemoryAddress.i
State.i
EndStructure
Structure lcmf_file_objects
LocalFileUUID.s
EndStructure
Structure lcmf_cache_servers
EndStructure
Structure lcmf_cache_servers_config_prepare
EndStructure
Procedure.i Main_Thread(ID.i)
;What this Thread does:
;* Check if Subthreads are alive and if not remove from Mapping Space
;* Spawn new Cache Threads and Kill Cache Threads
EndProcedure
Procedure.i Cache_Mem_Thread(ID.i)
;What this Thread does:
;* Manage the mem Objects
EndProcedure
Procedure.i Cache_File_Thread(ID.i)
;What this Thread does:
;* Manage the file Objects and maybe the memobjects.
EndProcedure
Procedure.i Copy_Thread(ID.i)
;What this Thread does:
;* Copy a File to Cachefolder
EndProcedure
Procedure.i Check_Thread(ID.i)
;What this Thread does:
;* Checks the Cachefolder and Memorymap if all Files are valid.
EndProcedure
Procedure.i Copy_File_to_Memory_Thread(ID.i)
;What this thread does:
;* Copy a local cached File to Memory
;
EndProcedure
EndModule

View file

@ -1,67 +1,229 @@
 ;********************************
;*
;* lweb_file_cache.pbi
;*
;* LiHaSo Webserver Modul In Memory File Cache (Obsolete and replaced by lcmf_main.pbi)
;*
;* It Could be configured Globaly and by Virtualhost
;*
DeclareModule lweb_FC
Enumeration Server_Type
#Type_Main ;Main Server
#Type_Virtual_Main ;Shared Virtual Server which get Data from Main
#Type_Virtual ;Standalone Virtual Server which has own Memory Area
EndEnumeration
;{ Error Enumeration
#Err_Unknown = -1
#Err_Server_Not_Found = -2
#Err_Server_File_Not_Found = -3
;}
Declare.s FC_Config_Get(UUID.s)
Declare.i FC_Config_Set(UUID.s, ConfigXML.s)
Declare.i FC_Create(UUID.s, Type.i = #Type_Main)
Declare.i FC_Destroy(UUID.s)
Declare.s FC_Status_Get(UUID.s)
Declare.i FC_GetFile(UUID.s, FileName.s)
Declare.i FC_SetFile(UUID.s, FileName.s, Memory.i, Size.i)
EndDeclareModule
Module lweb_FC
Structure St_Jobs
Type.i
Memory.i
EndStructure
Structure St_File
lifetime.i
timer.i
size.i
memoryID.i
ThreadID.i
Accesed.i
Mutex.i
Semaphore.i
List Job.St_Jobs()
EndStructure
Structure St_FC_Config
max_life_time.i
max_total_size.i
max_file_size.i
current_total_size.i
enabled.b
EndStructure
Structure St_Server
VServer_UUID.s
Log_UUID.s
Status.s
Thread.i
Semaphore.i
Config.St_FC_Config
Map Files.St_File()
EndStructure
Structure St_CacheServers
Main_UUID.s
Log_UUID.s
Status.s
Thread.i
Map Virtual.St_Server()
EndStructure
Global Var_FC_Server.St_CacheServers
Declare.i Cache_Cleaner(UUID.s)
Declare.i Cache_Server(UUID.s)
Declare.i Cache_File(UUID.s)
Procedure.s FC_Config_Get(UUID.s)
Protected XMLString.s, XMLID.i
If FindMapElement(FC\Virtual(), UUID)
XMLID = CreateXML(#PB_Any)
InsertXMLMap(RootXMLNode(XMLID), FC\Virtual\
Else
ProcedureReturn Str(#Err_Server_Not_Found)
EndIf
EndProcedure
Procedure.i FC_Config_Set(UUID.s, ConfigXML.s)
;Apply the Config XML to the Map
Protected XML_ID.i
XML_ID = CreateXML(#PB_Any)
If XMLStatus(XML_ID) = #PB_XML_Success
ParseXML(XML_ID, ConfigXML)
If XMLStatus(XML_ID) = #PB_XML_Success
ExtractXMLMap(MainXMLNode(XML_ID), @FC\Virtual(UUID), #PB_XML_NoCase)
FreeXML(XML_ID)
ProcedureReturn #True
Else
FreeXML(XML_ID)
ProcedureReturn #False
EndIf
Else
If XML_ID <> 0
FreeXML(XML_ID)
EndIf
ProcedureReturn #False
EndIf
EndProcedure
Procedure.i FC_Create(UUID.s, Type.i = #Type_Main)
;Create the Server
EndProcedure
Procedure.i FC_Destroy(UUID.s)
;Kill Thread Cache Handler Thread.
EndProcedure
Procedure.s FC_Status_Get(UUID.s)
;Status of the Server
EndEnumeration
Procedure.i FC_GetFile(UUID.s, FileName.s, MemoryID.i)
;Fill the Cached File to Memory and if response > 0 it was succesfull.
EndEnumeration
Procedure.i FC_SetFile(UUID.s, FileName.s, Memory.i, Size.i)
;Create cached file thread and save it to thread memory.
EndEnumeration
Procedure.i Cache_Cleaner(UUID.s)
;Cleans out old Files.
Protected.i Selected
Protected.i Counter
Protected.i MaxSize = Information
Repeat
Delay(1000) ;Nur Alle Sekunden Prüfen
Counter + 1
LockMutex(file_cache_mutex)
ResetMap(m_file_cache_map())
;Debug "FileCacheCleaner" + Str(Counter) + " Map Size:" + Str(MapSize(m_file_cache_map()))
If NextMapElement(m_file_cache_map())
Repeat
m_file_cache(MapKey(m_file_cache_map()))\Timer - 1
If m_file_cache(MapKey(m_file_cache_map()))\Timer <= 0
If m_file_cache(MapKey(m_file_cache_map()))\Buffer > 0
configuration\cache\current - m_file_cache(MapKey(m_file_cache_map()))\Size
FreeMemory(m_file_cache(MapKey(m_file_cache_map()))\Buffer)
DeleteMapElement(m_file_cache(), MapKey(m_file_cache_map()))
lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Killed:["+MapKey(m_file_cache_map())+"]")
Selected = DeleteMapElement(m_file_cache_map())
Else
lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Could Not kill:["+MapKey(m_file_cache_map())+"]")
EndIf
Else
Selected = NextMapElement(m_file_cache_map())
EndIf
Until Selected = 0
EndIf
UnlockMutex(file_cache_mutex)
ForEver
EndProcedure
Procedure.i Cache_Server(UUID.s)
;http://purearea.net/pb/english/manual/reference/ug_memory.html
Protected FileCacheCleanerThread.i
Protected Tempbuffer.i, MaxSize.i, Current.i
MaxSize = configuration\cache\maxsize * 1024 * 1024
FileCacheCleanerThread = CreateThread(@FileCacheCleaner(), MaxSize)
Repeat
WaitSemaphore(file_cache_semaphore)
lhs_log_ext::Out(configuration\log\CachelogUUID, "Adresse:"+m_file_cache()\Buffer)
If (m_file_cache()\Size + configuration\cache\current) <= MaxSize
configuration\cache\current + m_file_cache()\Size
Tempbuffer = AllocateMemory(m_file_cache()\Size)
CopyMemory(m_file_cache()\Buffer, Tempbuffer, MemorySize(m_file_cache()\Buffer))
m_file_cache()\Buffer = Tempbuffer
m_file_cache()\Timer = configuration\cache\time
m_file_cache()\Is = #True
m_file_cache_map(MapKey(m_file_cache())) = #True
SignalSemaphore(file_cache_semaphore_thread)
Else
lhs_log_ext::Out(configuration\log\CachelogUUID, "Cache Full")
SignalSemaphore(file_cache_semaphore_thread)
EndIf
ForEver
EndProcedure
Procedure.i Cache_File(UUID.s)
;Procedure for File_Cache Thread who handles the Cachefile
Protected MyMemory.i, My_Host_UUID.s = StringField(UUID, 0, ";"),My_File_UUID.s = StringField(UUID, 1, ";"), WorkElement.i
Repeat
WaitSemaphore(Var_FC_Server\Virtual(UUID)\Semaphore)
LockMutex(Var_FC_Server\Virtual(UUID)\Mutex)
WorkElement = FirstElement(Var_FC_Server\Virtual(UUID)\Files()
UnlockMutex(Var_FC_Server\Virtual(UUID)\Files(UUID)\Mutex)
ForEver
EndProcedure
EndModule
;Buggy FileCache Routines ... Wiso auch immer das noch nicht richtig Funktioniert. ;Buggy FileCache Routines ... Wiso auch immer das noch nicht richtig Funktioniert.
Procedure FileCacheCleaner(Information.i) Procedure FileCacheCleaner(Information.i)
Protected.i Selected
Protected.i Counter
Protected.i MaxSize = Information
Repeat
Delay(1000) ;Nur Alle Sekunden Prüfen
Counter + 1
LockMutex(file_cache_mutex)
ResetMap(m_file_cache_map())
;Debug "FileCacheCleaner" + Str(Counter) + " Map Size:" + Str(MapSize(m_file_cache_map()))
If NextMapElement(m_file_cache_map())
Repeat
m_file_cache(MapKey(m_file_cache_map()))\Timer - 1
If m_file_cache(MapKey(m_file_cache_map()))\Timer <= 0
If m_file_cache(MapKey(m_file_cache_map()))\Buffer > 0
configuration\cache\current - m_file_cache(MapKey(m_file_cache_map()))\Size
FreeMemory(m_file_cache(MapKey(m_file_cache_map()))\Buffer)
DeleteMapElement(m_file_cache(), MapKey(m_file_cache_map()))
lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Killed:["+MapKey(m_file_cache_map())+"]")
Selected = DeleteMapElement(m_file_cache_map())
Else
lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Could Not kill:["+MapKey(m_file_cache_map())+"]")
EndIf
Else
Selected = NextMapElement(m_file_cache_map())
EndIf
Until Selected = 0
EndIf
UnlockMutex(file_cache_mutex)
ForEver
EndProcedure EndProcedure
Procedure FileCache(Information.i) Procedure FileCache(Information.i)
;http://purearea.net/pb/english/manual/reference/ug_memory.html
Protected FileCacheCleanerThread.i
Protected Tempbuffer.i, MaxSize.i, Current.i
MaxSize = configuration\cache\maxsize * 1024 * 1024
FileCacheCleanerThread = CreateThread(@FileCacheCleaner(), MaxSize)
Repeat
WaitSemaphore(file_cache_semaphore)
lhs_log_ext::Out(configuration\log\CachelogUUID, "Adresse:"+m_file_cache()\Buffer)
If (m_file_cache()\Size + configuration\cache\current) <= MaxSize
configuration\cache\current + m_file_cache()\Size
Tempbuffer = AllocateMemory(m_file_cache()\Size)
CopyMemory(m_file_cache()\Buffer, Tempbuffer, MemorySize(m_file_cache()\Buffer))
m_file_cache()\Buffer = Tempbuffer
m_file_cache()\Timer = configuration\cache\time
m_file_cache()\Is = #True
m_file_cache_map(MapKey(m_file_cache())) = #True
SignalSemaphore(file_cache_semaphore_thread)
Else
lhs_log_ext::Out(configuration\log\CachelogUUID, "Cache Full")
SignalSemaphore(file_cache_semaphore_thread)
EndIf
ForEver
EndProcedure EndProcedure
Procedure AddFileToCache(MemoryID.i, FileName.s, Size.i) Procedure AddFileToCache(MemoryID.i, FileName.s, Size.i)
@ -92,10 +254,10 @@ Procedure.s GetFileFromCache(FileName.s, MemoryID.i)
String.s = #error_string String.s = #error_string
EndIf EndIf
UnlockMutex(file_cache_mutex) UnlockMutex(file_cache_mutex)
lhs_log_ext::Out(configuration\log\CachelogUUID, "File from Cache:"+FileName) lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Info, "File from Cache:"+FileName)
lhs_log_ext::Out(configuration\log\CachelogUUID, "Content: --------------------------------------------------------") lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Debugging, "Content: --------------------------------------------------------")
lhs_log_ext::Out(configuration\log\CachelogUUID, String) lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Debugging, String)
lhs_log_ext::Out(configuration\log\CachelogUUID, "End Content. ----------------------------------------------------") lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Debugging, "End Content. ----------------------------------------------------")
ProcedureReturn String.s ProcedureReturn String.s
Else Else
ProcedureReturn #error_string ProcedureReturn #error_string