Removal of old caching
This commit is contained in:
parent
72fc4436c9
commit
6a57a9f393
2 changed files with 352 additions and 61 deletions
129
inc/lcmf_main.pbi
Normal file
129
inc/lcmf_main.pbi
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
;********************************
|
||||
;*
|
||||
;* lcmf_main.pbi
|
||||
;*
|
||||
;* (c) Cache (m) Memory and (f) File
|
||||
;*
|
||||
;* LiHaSo Webserver Modul In Memory File Cache
|
||||
;*
|
||||
;* It Could be configured Globaly and by Virtualhost
|
||||
;*
|
||||
|
||||
DeclareModule lcmf
|
||||
#Cache_Server_Single = 0
|
||||
#Cache_Server_Multi = 1
|
||||
|
||||
EnumerationBinary lcmf_init
|
||||
#Mem_Cache
|
||||
#File_Cache
|
||||
EndEnumeration
|
||||
|
||||
EnumerationBinary lcmf_object_state
|
||||
#NA ;Not available
|
||||
#InProgress ;Copy is in Progress
|
||||
#Cached ;Object is available
|
||||
#Cleaning ;Object is on the way to cleaned out
|
||||
EndEnumeration
|
||||
|
||||
|
||||
EnumerationBinary lcmf_params_state
|
||||
;#UCP = Universal Cache Params (File and Mem)
|
||||
#UCP_Max_Size_Total ;Total Size of Cache
|
||||
#UCP_Max_Size_per_Entry ;Max Size of a File or Memoryblock
|
||||
|
||||
;#FAC = File Access Config
|
||||
#FAC_MC ;Minimum Access Times until a File is Cached (Generic)
|
||||
#FAC_RT ;Max Time a File could be in Cache (Generic)
|
||||
#FAC_LA_RT ;How long a File could stay in Cache sinde last Access (Generic)
|
||||
;(If a File has regulary access For ex. every 10 min. And this lifetime is 3600min. it will be killed after FAC_RT is reached)
|
||||
|
||||
#FAC_WL_Add ;Whitelist add Generic (Ex. :
|
||||
;".html" = All html files
|
||||
;"file.html" Expliciz Filenames
|
||||
;"/path/" Including all Files and Subdirectorys It is added to Valid_Source
|
||||
;"/path" only Files in Directory
|
||||
;"'/path/','.rpm'" Combines a path and a Specific file extension
|
||||
|
||||
#FAC_WL_Add_Ext ;Whitelist which add with special MC/RT/LA_RT Params
|
||||
#FAC_Valid_Source ;From where the files could get
|
||||
#FAC_Cache_Dir ;Directory where the Cached Files could be written
|
||||
|
||||
;Some Function for Set_State who do a active Work
|
||||
#FAC_Check_Cache ;Checks if all Files are the Same as Source
|
||||
#UCP_Flush_Object ;Removes a Object from Cache
|
||||
|
||||
EndEnumeration
|
||||
|
||||
Declare.s Init(Server_Type.i = #Cache_Server_Multi) ;Create a Main Cache_Thread Management Server
|
||||
Declare.s Init_cache(Cache_Type.i = #Mem_Cache) ;Create a Cache_Thread gives back the UUID
|
||||
Declare.i Set_Params(Server_ID.s, Parameters.s) ;Set Cache Parameters in pre run state
|
||||
Declare.i Use_New_Params(Server_ID.s) ;Reinit the Cache Thread
|
||||
Declare.i Run_Cache(Server_ID.s) ;Start a initialised Cache Init()->InitCache()->Set_Params()->Run_Cache()
|
||||
Declare.i Kill_Cache(Server_ID.s) ;Kills a Cache Server Thread
|
||||
Declare.i Set_State(Server_ID.s, Parameters.s) ;Force to do something in Cache Thread (Flush a Element, Cleanup the Filecache-IO Intesitive, Check for Updates on Files an co.)
|
||||
Declare.s Get_State(Server_ID.s, Parameters.s) ;Get Cache Server Thread State (For ex. current elements cachced, sizes and co.)
|
||||
|
||||
Declare.s Put_Mem(Server_ID.s, Memblock_ToSave.i, MC.i=-1, RT.i=-1, LA_RT.i=-1) ;Push a Memory Block to cache (Ex. some pre executed scripts or pictures and so on.)
|
||||
Declare.s Get_Mem(Server_ID.s, Memblock_ID.s, MC.i=-1, RT.i=-1, LA_RT.i=-1) ;Get That Memoryblock from cache who was Previously Stored
|
||||
Declare.s Get_File(Server_ID.s, Full_File_Name.s, Force_Disk.i = #True) ;Get a File (Cached or uncached) could be used everytime (ServerUUID empty everytime uncached or Flag uncached is set)
|
||||
|
||||
EndDeclareModule
|
||||
|
||||
Module lcmf
|
||||
|
||||
IncludeFile "lhs_uuid.pbi"
|
||||
|
||||
Structure lcmf_mem_objects
|
||||
MemoryAddress.i
|
||||
State.i
|
||||
EndStructure
|
||||
|
||||
Structure lcmf_file_objects
|
||||
LocalFileUUID.s
|
||||
|
||||
EndStructure
|
||||
|
||||
Structure lcmf_cache_servers
|
||||
|
||||
EndStructure
|
||||
|
||||
Structure lcmf_cache_servers_config_prepare
|
||||
|
||||
EndStructure
|
||||
|
||||
Procedure.i Main_Thread(ID.i)
|
||||
;What this Thread does:
|
||||
;* Check if Subthreads are alive and if not remove from Mapping Space
|
||||
;* Spawn new Cache Threads and Kill Cache Threads
|
||||
|
||||
EndProcedure
|
||||
|
||||
Procedure.i Cache_Mem_Thread(ID.i)
|
||||
;What this Thread does:
|
||||
;* Manage the mem Objects
|
||||
|
||||
EndProcedure
|
||||
|
||||
Procedure.i Cache_File_Thread(ID.i)
|
||||
;What this Thread does:
|
||||
;* Manage the file Objects and maybe the memobjects.
|
||||
EndProcedure
|
||||
|
||||
Procedure.i Copy_Thread(ID.i)
|
||||
;What this Thread does:
|
||||
;* Copy a File to Cachefolder
|
||||
EndProcedure
|
||||
|
||||
Procedure.i Check_Thread(ID.i)
|
||||
;What this Thread does:
|
||||
;* Checks the Cachefolder and Memorymap if all Files are valid.
|
||||
EndProcedure
|
||||
|
||||
Procedure.i Copy_File_to_Memory_Thread(ID.i)
|
||||
;What this thread does:
|
||||
;* Copy a local cached File to Memory
|
||||
;
|
||||
EndProcedure
|
||||
|
||||
EndModule
|
||||
|
||||
|
|
@ -1,67 +1,229 @@
|
|||
|
||||
;********************************
|
||||
;*
|
||||
;* lweb_file_cache.pbi
|
||||
;*
|
||||
;* LiHaSo Webserver Modul In Memory File Cache (Obsolete and replaced by lcmf_main.pbi)
|
||||
;*
|
||||
;* It Could be configured Globaly and by Virtualhost
|
||||
;*
|
||||
|
||||
DeclareModule lweb_FC
|
||||
Enumeration Server_Type
|
||||
#Type_Main ;Main Server
|
||||
#Type_Virtual_Main ;Shared Virtual Server which get Data from Main
|
||||
#Type_Virtual ;Standalone Virtual Server which has own Memory Area
|
||||
EndEnumeration
|
||||
|
||||
;{ Error Enumeration
|
||||
#Err_Unknown = -1
|
||||
#Err_Server_Not_Found = -2
|
||||
#Err_Server_File_Not_Found = -3
|
||||
;}
|
||||
|
||||
Declare.s FC_Config_Get(UUID.s)
|
||||
Declare.i FC_Config_Set(UUID.s, ConfigXML.s)
|
||||
Declare.i FC_Create(UUID.s, Type.i = #Type_Main)
|
||||
Declare.i FC_Destroy(UUID.s)
|
||||
Declare.s FC_Status_Get(UUID.s)
|
||||
Declare.i FC_GetFile(UUID.s, FileName.s)
|
||||
Declare.i FC_SetFile(UUID.s, FileName.s, Memory.i, Size.i)
|
||||
EndDeclareModule
|
||||
|
||||
Module lweb_FC
|
||||
Structure St_Jobs
|
||||
Type.i
|
||||
Memory.i
|
||||
EndStructure
|
||||
|
||||
Structure St_File
|
||||
lifetime.i
|
||||
timer.i
|
||||
size.i
|
||||
memoryID.i
|
||||
ThreadID.i
|
||||
Accesed.i
|
||||
Mutex.i
|
||||
Semaphore.i
|
||||
List Job.St_Jobs()
|
||||
EndStructure
|
||||
|
||||
Structure St_FC_Config
|
||||
max_life_time.i
|
||||
max_total_size.i
|
||||
max_file_size.i
|
||||
current_total_size.i
|
||||
enabled.b
|
||||
EndStructure
|
||||
|
||||
Structure St_Server
|
||||
VServer_UUID.s
|
||||
Log_UUID.s
|
||||
Status.s
|
||||
Thread.i
|
||||
Semaphore.i
|
||||
Config.St_FC_Config
|
||||
Map Files.St_File()
|
||||
EndStructure
|
||||
|
||||
Structure St_CacheServers
|
||||
Main_UUID.s
|
||||
Log_UUID.s
|
||||
Status.s
|
||||
Thread.i
|
||||
Map Virtual.St_Server()
|
||||
EndStructure
|
||||
|
||||
Global Var_FC_Server.St_CacheServers
|
||||
|
||||
Declare.i Cache_Cleaner(UUID.s)
|
||||
Declare.i Cache_Server(UUID.s)
|
||||
Declare.i Cache_File(UUID.s)
|
||||
|
||||
Procedure.s FC_Config_Get(UUID.s)
|
||||
Protected XMLString.s, XMLID.i
|
||||
If FindMapElement(FC\Virtual(), UUID)
|
||||
XMLID = CreateXML(#PB_Any)
|
||||
InsertXMLMap(RootXMLNode(XMLID), FC\Virtual\
|
||||
Else
|
||||
ProcedureReturn Str(#Err_Server_Not_Found)
|
||||
EndIf
|
||||
|
||||
EndProcedure
|
||||
|
||||
Procedure.i FC_Config_Set(UUID.s, ConfigXML.s)
|
||||
;Apply the Config XML to the Map
|
||||
Protected XML_ID.i
|
||||
XML_ID = CreateXML(#PB_Any)
|
||||
If XMLStatus(XML_ID) = #PB_XML_Success
|
||||
ParseXML(XML_ID, ConfigXML)
|
||||
If XMLStatus(XML_ID) = #PB_XML_Success
|
||||
ExtractXMLMap(MainXMLNode(XML_ID), @FC\Virtual(UUID), #PB_XML_NoCase)
|
||||
FreeXML(XML_ID)
|
||||
ProcedureReturn #True
|
||||
Else
|
||||
FreeXML(XML_ID)
|
||||
ProcedureReturn #False
|
||||
EndIf
|
||||
Else
|
||||
If XML_ID <> 0
|
||||
FreeXML(XML_ID)
|
||||
EndIf
|
||||
ProcedureReturn #False
|
||||
EndIf
|
||||
EndProcedure
|
||||
|
||||
Procedure.i FC_Create(UUID.s, Type.i = #Type_Main)
|
||||
;Create the Server
|
||||
EndProcedure
|
||||
|
||||
Procedure.i FC_Destroy(UUID.s)
|
||||
;Kill Thread Cache Handler Thread.
|
||||
EndProcedure
|
||||
|
||||
Procedure.s FC_Status_Get(UUID.s)
|
||||
;Status of the Server
|
||||
EndEnumeration
|
||||
|
||||
Procedure.i FC_GetFile(UUID.s, FileName.s, MemoryID.i)
|
||||
;Fill the Cached File to Memory and if response > 0 it was succesfull.
|
||||
EndEnumeration
|
||||
|
||||
Procedure.i FC_SetFile(UUID.s, FileName.s, Memory.i, Size.i)
|
||||
;Create cached file thread and save it to thread memory.
|
||||
EndEnumeration
|
||||
|
||||
Procedure.i Cache_Cleaner(UUID.s)
|
||||
;Cleans out old Files.
|
||||
Protected.i Selected
|
||||
Protected.i Counter
|
||||
Protected.i MaxSize = Information
|
||||
Repeat
|
||||
Delay(1000) ;Nur Alle Sekunden Prüfen
|
||||
Counter + 1
|
||||
LockMutex(file_cache_mutex)
|
||||
ResetMap(m_file_cache_map())
|
||||
;Debug "FileCacheCleaner" + Str(Counter) + " Map Size:" + Str(MapSize(m_file_cache_map()))
|
||||
If NextMapElement(m_file_cache_map())
|
||||
Repeat
|
||||
m_file_cache(MapKey(m_file_cache_map()))\Timer - 1
|
||||
If m_file_cache(MapKey(m_file_cache_map()))\Timer <= 0
|
||||
If m_file_cache(MapKey(m_file_cache_map()))\Buffer > 0
|
||||
configuration\cache\current - m_file_cache(MapKey(m_file_cache_map()))\Size
|
||||
FreeMemory(m_file_cache(MapKey(m_file_cache_map()))\Buffer)
|
||||
DeleteMapElement(m_file_cache(), MapKey(m_file_cache_map()))
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Killed:["+MapKey(m_file_cache_map())+"]")
|
||||
Selected = DeleteMapElement(m_file_cache_map())
|
||||
Else
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Could Not kill:["+MapKey(m_file_cache_map())+"]")
|
||||
EndIf
|
||||
Else
|
||||
Selected = NextMapElement(m_file_cache_map())
|
||||
EndIf
|
||||
Until Selected = 0
|
||||
EndIf
|
||||
|
||||
UnlockMutex(file_cache_mutex)
|
||||
|
||||
|
||||
ForEver
|
||||
EndProcedure
|
||||
|
||||
Procedure.i Cache_Server(UUID.s)
|
||||
;http://purearea.net/pb/english/manual/reference/ug_memory.html
|
||||
Protected FileCacheCleanerThread.i
|
||||
Protected Tempbuffer.i, MaxSize.i, Current.i
|
||||
MaxSize = configuration\cache\maxsize * 1024 * 1024
|
||||
FileCacheCleanerThread = CreateThread(@FileCacheCleaner(), MaxSize)
|
||||
|
||||
Repeat
|
||||
WaitSemaphore(file_cache_semaphore)
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "Adresse:"+m_file_cache()\Buffer)
|
||||
If (m_file_cache()\Size + configuration\cache\current) <= MaxSize
|
||||
configuration\cache\current + m_file_cache()\Size
|
||||
Tempbuffer = AllocateMemory(m_file_cache()\Size)
|
||||
CopyMemory(m_file_cache()\Buffer, Tempbuffer, MemorySize(m_file_cache()\Buffer))
|
||||
m_file_cache()\Buffer = Tempbuffer
|
||||
m_file_cache()\Timer = configuration\cache\time
|
||||
m_file_cache()\Is = #True
|
||||
m_file_cache_map(MapKey(m_file_cache())) = #True
|
||||
SignalSemaphore(file_cache_semaphore_thread)
|
||||
Else
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "Cache Full")
|
||||
SignalSemaphore(file_cache_semaphore_thread)
|
||||
EndIf
|
||||
|
||||
ForEver
|
||||
EndProcedure
|
||||
|
||||
Procedure.i Cache_File(UUID.s)
|
||||
;Procedure for File_Cache Thread who handles the Cachefile
|
||||
Protected MyMemory.i, My_Host_UUID.s = StringField(UUID, 0, ";"),My_File_UUID.s = StringField(UUID, 1, ";"), WorkElement.i
|
||||
Repeat
|
||||
WaitSemaphore(Var_FC_Server\Virtual(UUID)\Semaphore)
|
||||
LockMutex(Var_FC_Server\Virtual(UUID)\Mutex)
|
||||
WorkElement = FirstElement(Var_FC_Server\Virtual(UUID)\Files()
|
||||
|
||||
UnlockMutex(Var_FC_Server\Virtual(UUID)\Files(UUID)\Mutex)
|
||||
|
||||
|
||||
ForEver
|
||||
|
||||
|
||||
EndProcedure
|
||||
|
||||
|
||||
EndModule
|
||||
|
||||
|
||||
;Buggy FileCache Routines ... Wiso auch immer das noch nicht richtig Funktioniert.
|
||||
|
||||
Procedure FileCacheCleaner(Information.i)
|
||||
Protected.i Selected
|
||||
Protected.i Counter
|
||||
Protected.i MaxSize = Information
|
||||
Repeat
|
||||
Delay(1000) ;Nur Alle Sekunden Prüfen
|
||||
Counter + 1
|
||||
LockMutex(file_cache_mutex)
|
||||
ResetMap(m_file_cache_map())
|
||||
;Debug "FileCacheCleaner" + Str(Counter) + " Map Size:" + Str(MapSize(m_file_cache_map()))
|
||||
If NextMapElement(m_file_cache_map())
|
||||
Repeat
|
||||
m_file_cache(MapKey(m_file_cache_map()))\Timer - 1
|
||||
If m_file_cache(MapKey(m_file_cache_map()))\Timer <= 0
|
||||
If m_file_cache(MapKey(m_file_cache_map()))\Buffer > 0
|
||||
configuration\cache\current - m_file_cache(MapKey(m_file_cache_map()))\Size
|
||||
FreeMemory(m_file_cache(MapKey(m_file_cache_map()))\Buffer)
|
||||
DeleteMapElement(m_file_cache(), MapKey(m_file_cache_map()))
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Killed:["+MapKey(m_file_cache_map())+"]")
|
||||
Selected = DeleteMapElement(m_file_cache_map())
|
||||
Else
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "Info:["+Str(innercount)+"] Could Not kill:["+MapKey(m_file_cache_map())+"]")
|
||||
EndIf
|
||||
Else
|
||||
Selected = NextMapElement(m_file_cache_map())
|
||||
EndIf
|
||||
Until Selected = 0
|
||||
EndIf
|
||||
|
||||
UnlockMutex(file_cache_mutex)
|
||||
|
||||
|
||||
ForEver
|
||||
|
||||
|
||||
EndProcedure
|
||||
|
||||
Procedure FileCache(Information.i)
|
||||
;http://purearea.net/pb/english/manual/reference/ug_memory.html
|
||||
Protected FileCacheCleanerThread.i
|
||||
Protected Tempbuffer.i, MaxSize.i, Current.i
|
||||
MaxSize = configuration\cache\maxsize * 1024 * 1024
|
||||
FileCacheCleanerThread = CreateThread(@FileCacheCleaner(), MaxSize)
|
||||
|
||||
Repeat
|
||||
WaitSemaphore(file_cache_semaphore)
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "Adresse:"+m_file_cache()\Buffer)
|
||||
If (m_file_cache()\Size + configuration\cache\current) <= MaxSize
|
||||
configuration\cache\current + m_file_cache()\Size
|
||||
Tempbuffer = AllocateMemory(m_file_cache()\Size)
|
||||
CopyMemory(m_file_cache()\Buffer, Tempbuffer, MemorySize(m_file_cache()\Buffer))
|
||||
m_file_cache()\Buffer = Tempbuffer
|
||||
m_file_cache()\Timer = configuration\cache\time
|
||||
m_file_cache()\Is = #True
|
||||
m_file_cache_map(MapKey(m_file_cache())) = #True
|
||||
SignalSemaphore(file_cache_semaphore_thread)
|
||||
Else
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "Cache Full")
|
||||
SignalSemaphore(file_cache_semaphore_thread)
|
||||
EndIf
|
||||
|
||||
ForEver
|
||||
|
||||
EndProcedure
|
||||
|
||||
Procedure AddFileToCache(MemoryID.i, FileName.s, Size.i)
|
||||
|
|
@ -92,10 +254,10 @@ Procedure.s GetFileFromCache(FileName.s, MemoryID.i)
|
|||
String.s = #error_string
|
||||
EndIf
|
||||
UnlockMutex(file_cache_mutex)
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "File from Cache:"+FileName)
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "Content: --------------------------------------------------------")
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, String)
|
||||
lhs_log_ext::Out(configuration\log\CachelogUUID, "End Content. ----------------------------------------------------")
|
||||
lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Info, "File from Cache:"+FileName)
|
||||
lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Debugging, "Content: --------------------------------------------------------")
|
||||
lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Debugging, String)
|
||||
lhs_log_ext::OutL(configuration\log\CachelogUUID,lhs_log_ext::#Debugging, "End Content. ----------------------------------------------------")
|
||||
ProcedureReturn String.s
|
||||
Else
|
||||
ProcedureReturn #error_string
|
||||
|
|
|
|||
Loading…
Reference in a new issue