init-commit

README.md

@@ -1,3 +1,59 @@
 # fsexec
 
-ArOZ Online System File Explorer - Freeze Script Execution Application, design for replacing those script that was freezing the system while file IO.
+ArOZ Online System File Explorer - Freeze Script Execution Application, design for replacing those script that was freezing the system while file IO.
+
+## What is Freeze Script Execution Application?
+Freeze Script Execution Application - Short for fsexec, is a binary written in golang to replace the old scripts in the ArOZ Online File System that would freeze the whole PHP server during file IO operations.
+This script make use of golang's super fast IO and processing ability and make file operation much faster on the new implementation.
+
+The fsexec application will receive the command as two parts. In most case, you will be using these funcitons only.
+
+./fsexec {uuid} {command}
+
+where command is a string in base64 encoded JSON string.
+
+## So, how can I use this binary?
+
+For example, you want to use the funciton [copy] with paramter {source} and {target}.
+
+First, you need to convert the command into an array and parse it as JSON string.
+
+For example, you have the following command array:
+```
+["copy","./test.txt","target/test.txt"]
+```
+Next, you would want to stringify it as JSON string.
+```
+"[\"copy\",\"./test.txt\",\"target/test.txt\"]"
+```
+Then, you can convert the string above into base64 representation. This is what you will get:
+```
+WyJjb3B5IiwiLi90ZXN0LnR4dCIsInRhcmdldC90ZXN0LnR4dCJd
+```
+Lastly, you call the fsexec with the following command. Assumeing the {uuid} is [123]
+```
+./fsexec 123 WyJjb3B5IiwiLi90ZXN0LnR4dCIsInRhcmdldC90ZXN0LnR4dCJd
+```
+
+Here are some examples:
+```
+# copy test.txt test/test.txt
+./fsexec uuid_string WyJjb3B5IiwidGVzdC50eHQiLCJmb2xkZXIvdGVzdC50eHQiXQ==
+
+# copy_folder test/ target/test/
+./fsexec uuid_string WyJjb3B5X2ZvbGRlciIsInRlc3QvIiwidGFyZ2V0L3Rlc3QvIl0=
+```
+The fsexec application will create a folder named "log" if it doesn't exists.
+Inside the log folder, you would found "done" and "error" sub-folders. In simple words:
+
+- For all successfully finished file operation, its log will be stored in "done" directory.
+- For all failed file operation, its log will be stored in "error" directory.
+- For all processing file operation, its log will be stored under "log/" directory but not its subfolders.
+
+## What is UUID then?
+As its name suggest, UUID is a unique string or interger that can be used to identify each file operations. 
+You can use anything you want as soon as it is unique. Just to make sure don't use some wierd, not universal-supported characters.
+
+## License
+CopyRight Author Toby Chui under ArOZ Online Project, 2019
+

examples.txt

@@ -0,0 +1,6 @@
+Exmaple usage:
+<< copy test.txt test/test.txt >>
+./fsexec {uuid} WyJjb3B5IiwidGVzdC50eHQiLCJmb2xkZXIvdGVzdC50eHQiXQ==
+
+<< copy_folder test/ target/test/ >>
+./fsexec {uuid} WyJjb3B5X2ZvbGRlciIsInRlc3QvIiwidGFyZ2V0L3Rlc3QvIl0=

log/done/100.log

@@ -0,0 +1,3 @@
+[init] 2019/08/12 14:19:57 Task created on 2019-08-12 141957
+[info] 2019/08/12 14:19:57 ["move","test.txt","target/test.txt"]
+[done] 2019/08/12 14:19:57 Task finished successfully

log/error/0001.log

@@ -0,0 +1,3 @@
+[init] 2019/08/12 14:10:53 Task created on 2019-08-12 141053
+[info] 2019/08/12 14:10:53 ["copy_folder","test/","target/test/"]
+[error] 2019/08/12 14:10:53 Invalid source file or target directory.

main.go

@@ -0,0 +1,349 @@
+/*
+ArOZ Online System - fsexec File System Asynchronize File Opertation Execution Instance
+
+This is a stand alone application design to replace the synchronize file operation designed since the start
+of the AOB project. By replacing the original file operation php script with this new script, the file explorer
+in AOB should be "freeze-free" while moving / copying large files.
+
+This application provide the following functions
+
+copy {from} {target}
+copy_folder {from} {target}
+delete {filename}
+move {from} {target}
+move_folder {from} {target}
+
+All the command should be stringify with JSON and encoded into base64, then pass into the command parameter of the
+application launching paramters.
+*/
+
+package main
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"reflect"
+	"time"
+)
+
+//Commonly used functions
+func in_array(val interface{}, array interface{}) (exists bool, index int) {
+	exists = false
+	index = -1
+
+	switch reflect.TypeOf(array).Kind() {
+	case reflect.Slice:
+		s := reflect.ValueOf(array)
+
+		for i := 0; i < s.Len(); i++ {
+			if reflect.DeepEqual(val, s.Index(i).Interface()) == true {
+				index = i
+				exists = true
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func file_exists(path string) bool {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func file_get_contents(path string) string {
+	dat, err := ioutil.ReadFile(path)
+	if err != nil {
+		panic("Unable to read file: " + path)
+	}
+	return (string(dat))
+
+}
+
+func writeLog(filepath string, prefix string, message string) bool {
+	f, err := os.OpenFile(filepath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		log.Println(err)
+	}
+	defer f.Close()
+
+	logger := log.New(f, prefix, log.LstdFlags)
+	logger.Println(message)
+	return true
+}
+
+func showHelp() {
+	fmt.Println("<<ArOZ Online System File System Freeze Script Executer>>")
+	fmt.Println("Usage: ./fsexec {uuid} {command_in_base64}")
+}
+
+func decodeBase64(base64string string) string {
+	l, err := base64.StdEncoding.DecodeString(base64string)
+	if err != nil {
+		panic(err)
+	}
+	return string(l)
+}
+
+func validateCopySourceAndTarget(from string, target string) bool {
+	if !file_exists(from) {
+		return false
+	}
+	if file_exists(target) {
+		return false
+	}
+	return true
+}
+
+func getTimestamp() string {
+	t := time.Now()
+	return (t.Format("2006-01-02 150405"))
+}
+
+func moveLogFile(logfile string, target string, uuid string) {
+	//If then else is used to prevent invalid move target
+	if target == "done" {
+		os.Rename(logfile, "log/done/"+uuid+".log")
+	} else if target == "error" {
+		os.Rename(logfile, "log/error/"+uuid+".log")
+	} else {
+		//What happened?o
+		panic("ERROR. Undefined log file sattle location.")
+	}
+
+}
+
+func finishFileOperation(logfile string, uuid string) {
+	//Finishing the file operation.
+	writeLog(logfile, "[done] ", "Task finished successfully")
+	moveLogFile(logfile, "done", uuid)
+	fmt.Println("Done")
+}
+
+func initChk() {
+	//Initiation Checking for all the required directories
+	if !file_exists("log/") {
+		os.MkdirAll("log/", 0777)
+	}
+	if !file_exists("log/done/") {
+		os.MkdirAll("log/done/", 0777)
+	}
+	if !file_exists("log/error/") {
+		os.MkdirAll("log/error/", 0777)
+	}
+}
+
+//Main programming logic
+func main() {
+	initChk()
+	acceptOperations := []string{"copy", "copy_folder", "delete", "move", "move_folder"}
+	if len(os.Args) == 1 {
+		showHelp()
+		return
+	}
+	//Get the command in using base64 and required log file uuid
+	startSettings := os.Args[1:]
+	uuid := startSettings[0]
+	encodedCommand := startSettings[1]
+	decodedCommand := decodeBase64(encodedCommand)
+	filename := uuid + ".log"
+	var command []string
+	//Create the log file for this task
+	logfile := "log/" + filename
+	//Check if the given id exists already.
+	if file_exists(logfile) || file_exists("log/done/"+filename) || file_exists("log/error/"+filename) {
+		panic("ERROR. Required task already exists.")
+	}
+	//Create the log file for this task
+	writeLog(logfile, "[init] ", "Task created on "+getTimestamp())
+
+	err := json.Unmarshal([]byte(decodedCommand), &command)
+	if err != nil {
+		writeLog(logfile, "[error] ", "Unable to parse JSON string.")
+		moveLogFile(logfile, "error", uuid)
+		panic("ERROR. Unable to parse JSON string.")
+	}
+	//Check if the given file operation correct or not.
+	correctOpr, _ := in_array(string(command[0]), acceptOperations)
+	if !correctOpr {
+		fmt.Println(string(command[0]))
+		writeLog(logfile, "[error] ", "Invalid file operation. "+command[0]+" given.")
+		moveLogFile(logfile, "error", uuid)
+		panic("ERROR. Invalid file operation. " + command[0] + " given.")
+	}
+
+	writeLog(logfile, "[info] ", decodedCommand)
+	//Start reading the command for file operations
+	opr := command[0]
+	if opr == "copy" {
+		//Copy a given file to a given location
+		from := command[1]
+		target := command[2]
+		if !validateCopySourceAndTarget(from, target) {
+			writeLog(logfile, "[error] ", "Invalid source file or target directory.")
+			moveLogFile(logfile, "error", uuid)
+			panic("ERROR. Invalid source file or target directory.")
+		}
+		//Check if the parent dir exists
+		dirname := path.Dir(target)
+		if !file_exists(dirname) {
+			os.MkdirAll(dirname, 0777)
+		}
+		Copy(from, target)
+		finishFileOperation(logfile, uuid)
+	} else if opr == "copy_folder" {
+		//Copy a folder to a given location
+		from := command[1]
+		target := command[2]
+		if !validateCopySourceAndTarget(from, target) {
+			writeLog(logfile, "[error] ", "Invalid source file or target directory.")
+			moveLogFile(logfile, "error", uuid)
+			panic("ERROR. Invalid source file or target directory.")
+		}
+		Copy(from, target)
+		finishFileOperation(logfile, uuid)
+	} else if opr == "move" {
+		//Move a file
+		from := command[1]
+		target := command[2]
+		if !validateCopySourceAndTarget(from, target) {
+			writeLog(logfile, "[error] ", "Invalid source file or target directory.")
+			moveLogFile(logfile, "error", uuid)
+			panic("ERROR. Invalid source file or target directory.")
+		}
+		err := os.Rename(from, target)
+		if err != nil {
+			writeLog(logfile, "[error] ", "Unable to move file due to unknown error.")
+			moveLogFile(logfile, "error", uuid)
+			panic("ERROR. Unable to move file due to unknown error.")
+		}
+		finishFileOperation(logfile, uuid)
+	} else if opr == "move_folder" {
+		//Move a folder
+		from := command[1]
+		target := command[2]
+		if !validateCopySourceAndTarget(from, target) {
+			writeLog(logfile, "[error] ", "Invalid source file or target directory.")
+			moveLogFile(logfile, "error", uuid)
+			panic("ERROR. Invalid source file or target directory.")
+		}
+		err := os.Rename(from, target)
+		if err != nil {
+			writeLog(logfile, "[error] ", "Unable to move folder due to unknown error.")
+			moveLogFile(logfile, "error", uuid)
+			panic("ERROR. Unable to move folder due to unknown error.")
+		}
+		finishFileOperation(logfile, uuid)
+	}
+
+}
+
+//Program required external library. Included in main to prevent download on other building platform.
+//Recursive Copy Lib, not sure why it can't be imported like other module. Hence directly copy and paste in the section below.
+const (
+	// tmpPermissionForDirectory makes the destination directory writable,
+	// so that stuff can be copied recursively even if any original directory is NOT writable.
+	// See https://github.com/otiai10/copy/pull/9 for more information.
+	tmpPermissionForDirectory = os.FileMode(0755)
+)
+
+// Copy copies src to dest, doesn't matter if src is a directory or a file
+func Copy(src, dest string) error {
+	info, err := os.Lstat(src)
+	if err != nil {
+		return err
+	}
+	return copy(src, dest, info)
+}
+
+// copy dispatches copy-funcs according to the mode.
+// Because this "copy" could be called recursively,
+// "info" MUST be given here, NOT nil.
+func copy(src, dest string, info os.FileInfo) error {
+	if info.Mode()&os.ModeSymlink != 0 {
+		return lcopy(src, dest, info)
+	}
+	if info.IsDir() {
+		return dcopy(src, dest, info)
+	}
+	return fcopy(src, dest, info)
+}
+
+// fcopy is for just a file,
+// with considering existence of parent directory
+// and file permission.
+func fcopy(src, dest string, info os.FileInfo) error {
+
+	if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil {
+		return err
+	}
+
+	f, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if err = os.Chmod(f.Name(), info.Mode()); err != nil {
+		return err
+	}
+
+	s, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	_, err = io.Copy(f, s)
+	return err
+}
+
+// dcopy is for a directory,
+// with scanning contents inside the directory
+// and pass everything to "copy" recursively.
+func dcopy(srcdir, destdir string, info os.FileInfo) error {
+
+	originalMode := info.Mode()
+
+	// Make dest dir with 0755 so that everything writable.
+	if err := os.MkdirAll(destdir, tmpPermissionForDirectory); err != nil {
+		return err
+	}
+	// Recover dir mode with original one.
+	defer os.Chmod(destdir, originalMode)
+
+	contents, err := ioutil.ReadDir(srcdir)
+	if err != nil {
+		return err
+	}
+
+	for _, content := range contents {
+		cs, cd := filepath.Join(srcdir, content.Name()), filepath.Join(destdir, content.Name())
+		if err := copy(cs, cd, content); err != nil {
+			// If any error, exit immediately
+			return err
+		}
+	}
+
+	return nil
+}
+
+// lcopy is for a symlink,
+// with just creating a new symlink by replicating src symlink.
+func lcopy(src, dest string, info os.FileInfo) error {
+	src, err := os.Readlink(src)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(src, dest)
+}