diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml new file mode 100644 index 00000000..46251abb --- /dev/null +++ b/.github/workflows/dev.yaml @@ -0,0 +1,76 @@ +name: dev + +on: + push: + branches: + - 'dev_v*' + +jobs: + test-backend: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: 1.21.1 + - run: cd backend && go test -race -v ./... + lint-backend: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: 1.21.1 + - run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2 + - run: cd backend && golangci-lint run + format-backend: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: 1.21.1 + - run: cd backend && go fmt ./... + lint-frontend: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '20' + - run: cd frontend && npm i eslint + - run: cd frontend && npm run lint + push_dev_to_registry: + name: Push dev image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + # Workaround to fix error: + # failed to push: failed to copy: io: read/write on closed pipe + # See https://github.com/docker/build-push-action/issues/761 + with: + driver-opts: | + image=moby/buildkit:v0.10.6 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: gtstef/filebrowser + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/pr-merge.yaml b/.github/workflows/pr-merge.yaml index bfc65e06..a3204b56 100644 --- a/.github/workflows/pr-merge.yaml +++ b/.github/workflows/pr-merge.yaml @@ -4,6 +4,7 @@ on: pull_request: branches: - 'main' + - 'dev_v*' - 'v[0-9]+.[0-9]+.[0-9]+' jobs: @@ -43,7 +44,6 @@ jobs: - run: cd frontend && npm run lint push_pr_to_registry: - needs: [lint-frontend, lint-backend, test-backend, format-backend] name: Push PR runs-on: ubuntu-latest steps: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e620ec77..5061501f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,6 +4,7 @@ on: push: branches: - 'v[0-9]+.[0-9]+.[0-9]+' + jobs: test-backend: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bfbe207..bf409e12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,69 +1,88 @@ # Changelog -All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +All notable changes to this project will be documented in this file. For commit guidelines, please refer to [Standard Version](https://github.com/conventional-changelog/standard-version). -# v0.2.1 - - issue #29 - Rules can now be configured and read from configuration yaml - - issue #28 - Allow disable settings per user. - - issue #27 - shorten download link for password protected files - - issue #26 - enable dark mode per user and improve switching performance. - - More rounded corners and improved listing styling - - improve search performance - - fixes authentication issues - - adds compact view mode - - improves view mode configuration and behavior - - updates configuration file to accept new settings +## v0.2.2 -# v0.2.0 - - improved UI - - more unified coehisive look - - Adjusted header bar look and icon behavior - - The shell is dead. - - If you need to use custom commands, exec into the docker container. - - The json config file is dead. - - All configuration is done via advanced `filebrowser.yaml` - - The only flag that is allowed is flag to specify config file. - - Removed old code to migrate database versions - - Removed all unused cmd code +- **Major Indexing Changes:** + - **Speed:** (0m57s) - Decreased by 78% compared to the previous release. + - **Memory Usage:** (41MB) - Reduced by 45% compared to the previous release. +- Now utilizes the index for file browser listings! +- **[Work in Progress]** Hidden files are still directly accessible. +- **[Work in Progress]** Editor issues fixed on save and themes. +- **[Work in Progress]** `running-config.yaml` gets updated when settings change, ensuring that running settings are up to date. -# v0.1.4 - - various UI fixes - - Added download button back to toolbar - - Added upload button to side menu - - breadcrumb spacing fix - - Added "compact" view option - - fixed slash issue with css rtl logic - - various backend fixes - - search has a sessionId attached so searches don't collide - - search no longer searches by word with spaces, includes space in searches - - prepared for full json configuration - - made size search work for smaller and larger - - made search types not show up in search bar when used +## v0.2.1 + +- Addressed issue #29 - Rules can now be configured and read from the configuration YAML. +- Addressed issue #28 - Allows disabling settings per user. +- Addressed issue #27 - Shortened download link for password-protected files. +- Addressed issue #26 - Enables dark mode per user and improves switching performance. +- Improved styling with more rounded corners and enhanced listing design. +- Enhanced search performance. +- Fixed authentication issues. +- Added compact view mode. +- Improved view mode configuration and behavior. +- Updated the configuration file to accept new settings. + +## v0.2.0 + +- **Improved UI:** + - Enhanced the cohesive and unified look. + - Adjusted the header bar appearance and icon behavior. +- The shell feature has been deprecated. + - Custom commands can be executed within the Docker container if needed. +- The JSON config file is no longer used. + - All configurations are now performed via the advanced `filebrowser.yaml`. + - The only allowed flag is specifying the config file. +- Removed old code for migrating database versions. +- Eliminated all unused `cmd` code. + +## v0.1.4 + +- **Various UI fixes:** + - Reintroduced the download button to the toolbar. + - Added the upload button to the side menu. + - Adjusted breadcrumb spacing. + - Introduced a "compact" view option. + - Fixed a slash issue with CSS right-to-left (RTL) logic. +- **Various backend improvements:** + - Added session IDs to searches to prevent collisions. + - Modified search behavior to include spaces in searches. + - Prepared for full JSON configuration support. +- Made size-based searches work for both smaller and larger files. +- Modified search types not to appear in the search bar when used. ## v0.1.3 - - improved styling, colors, transparency, blur - - Made sidebar hidden on desktop as well - - simplified navbar to be three buttons - - open menu - - search - - toggle view - - Changed desktop search style and included additional search options. +- Enhanced styling with improved colors, transparency, and blur effects. +- Hid the sidebar on desktop views. +- Simplified the navbar to include three buttons: + - Open menu + - Search + - Toggle view +- Revised desktop search style and included additional search options. ## v0.1.2 - - Updated UI to use search features better - - More filter options - - Better icons with colors - - GUI styling - - Improved search performance +- Updated the UI to better utilize search features: + - Added more filter options. + - Enhanced icons with colors. + - Improved GUI styling. +- Improved search performance. +- **Index Changes:** + - **Speed:** (0m32s) - Increased by 6% compared to the previous release. + - **Memory Usage:** (93MB) - Increased by 3% compared to the previous release. ## v0.1.1 - - Improved search with indexing +- Improved search functionality with indexing. +- **Index Changes (Baseline Results):** + - **Speed:** (0m30s) + - **Memory Usage:** (90MB) ## v0.1.0 - - nothing changed from origin. +- No changes from the original. -Forked from https://github.com/filebrowser/filebrowser +Forked from [filebrowser/filebrowser](https://github.com/filebrowser/filebrowser). diff --git a/README.md b/README.md index 21778894..cff698b1 100644 --- a/README.md +++ b/README.md @@ -9,21 +9,25 @@

-> **NOTE** -Intended for docker use only +> [!NOTE] +> Only intended to be used with docker. -> **Warning** -Starting with v0.2.0, *ALL* configuration is done via `filebrowser.yaml` configuration file. `.filebrowser.json` and any flags other than `-c` and `-config` during execution WILL NO LONGER WORK. This is by design, in order to use the v0.2.0 You can mount your directory and initialize a new DB with a new default `filebrowser.yaml` which you can tweak and use in the future. Or you can copy and paste the default startup `filebrowser.yaml` below. +> [!WARNING] +> Starting with v0.2.0, *ALL* configuration is done via `filebrowser.yaml` configuration file. This fork makes the following significant changes to filebrowser for origin: - 1. [x] Improves search to use index instead of filesystem. - - Lightning fast, realtime results as you type + 1. [x] Better search. + - Lightning fast + - realtime results as you type - Works with more type filters - 1. [x] Improved and simplified GUI navbar and sidebar menu. - 1. [x] Updated version and dependencies. - 1. [x] **IMPORTANT** Moved all configurations to `filebrowser.yaml`. - + - interactive results page. + 1. [x] Revamped and simplified GUI navbar and sidebar menu. + 1. [x] **IMPORTANT** Revamped configuration via `filebrowser.yml` config file. + 1. [x] More configurations possible at a per-user level + - image + 1. [x] Additional compact view mode as well as refreshed view mode styles. + ## About Filebrowser provides a file managing interface within a specified directory @@ -47,26 +51,6 @@ work better in terms of asthetics and performance. Improved search,

-## Search Performance - -100x faster search. However, this will be at expense of RAM. if you have < 1 million -files and folders in the given scope, the RAM usage should be less than 200MB total. RAM requirements -should scale based on the number of directories. - -Also , the approx. time to fully index will vary widely based on performance. A sufficiently performant -system should fully index within the first 5 minutes, potentially within the first few seconds. - -For example, a low end 11th gen i5 with SSD indexes 128K files within 1 second: - -``` -2023/09/09 21:38:50 Initializing with config file: filebrowser.yaml -2023/09/09 21:38:50 Indexing files... -2023/09/09 21:38:50 Listening on [::]:8080 -2023/09/09 21:38:51 Successfully indexed files. -2023/09/09 21:38:51 Files found : 123452 -2023/09/09 21:38:51 Directories found : 1768 -2023/09/09 21:38:51 Indexing scheduler will run every 5 minutes -``` ## Install diff --git a/backend/auth/hook.go b/backend/auth/hook.go index c1902947..e113f355 100644 --- a/backend/auth/hook.go +++ b/backend/auth/hook.go @@ -10,7 +10,6 @@ import ( "strings" "github.com/gtsteffaniak/filebrowser/errors" - "github.com/gtsteffaniak/filebrowser/files" "github.com/gtsteffaniak/filebrowser/settings" "github.com/gtsteffaniak/filebrowser/users" ) @@ -44,8 +43,8 @@ func (a *HookAuth) Auth(r *http.Request, usr users.Store) (*users.User, error) { } a.Users = usr - a.Settings = &settings.GlobalConfiguration - a.Server = &settings.GlobalConfiguration.Server + a.Settings = &settings.Config + a.Server = &settings.Config.Server a.Cred = cred action, err := a.RunCommand() @@ -207,7 +206,7 @@ func (a *HookAuth) GetUser(d *users.User) *users.User { Locale: d.Locale, ViewMode: d.ViewMode, SingleClick: d.SingleClick, - Sorting: files.Sorting{ + Sorting: users.Sorting{ Asc: d.Sorting.Asc, By: d.Sorting.By, }, diff --git a/backend/auth/json.go b/backend/auth/json.go index 7344cd04..e05547de 100644 --- a/backend/auth/json.go +++ b/backend/auth/json.go @@ -24,7 +24,7 @@ type JSONAuth struct { // Auth authenticates the user via a json in content body. func (a JSONAuth) Auth(r *http.Request, usr users.Store) (*users.User, error) { - config := &settings.GlobalConfiguration + config := &settings.Config var cred jsonCred if r.Body == nil { diff --git a/backend/auth/none.go b/backend/auth/none.go index 43688a26..0d6a9691 100644 --- a/backend/auth/none.go +++ b/backend/auth/none.go @@ -15,7 +15,7 @@ type NoAuth struct{} // Auth uses authenticates user 1. func (a NoAuth) Auth(r *http.Request, usr users.Store) (*users.User, error) { - return usr.Get(settings.GlobalConfiguration.Server.Root, uint(1)) + return usr.Get(settings.Config.Server.Root, uint(1)) } // LoginPage tells that no auth doesn't require a login page. diff --git a/backend/auth/proxy.go b/backend/auth/proxy.go index cf3e5936..522618b0 100644 --- a/backend/auth/proxy.go +++ b/backend/auth/proxy.go @@ -21,7 +21,7 @@ type ProxyAuth struct { // Auth authenticates the user via an HTTP header. func (a ProxyAuth) Auth(r *http.Request, usr users.Store) (*users.User, error) { username := r.Header.Get(a.Header) - user, err := usr.Get(settings.GlobalConfiguration.Server.Root, username) + user, err := usr.Get(settings.Config.Server.Root, username) if err == errors.ErrNotExist { return nil, os.ErrPermission } diff --git a/backend/benchmark_results.txt b/backend/benchmark_results.txt index 029e0805..d6e096e5 100644 --- a/backend/benchmark_results.txt +++ b/backend/benchmark_results.txt @@ -5,41 +5,40 @@ ? github.com/gtsteffaniak/filebrowser/auth [no test files] ? github.com/gtsteffaniak/filebrowser/cmd [no test files] PASS -ok github.com/gtsteffaniak/filebrowser/diskcache 0.004s +ok github.com/gtsteffaniak/filebrowser/diskcache 0.003s ? github.com/gtsteffaniak/filebrowser/errors [no test files] -? github.com/gtsteffaniak/filebrowser/files [no test files] +goos: linux +goarch: amd64 +pkg: github.com/gtsteffaniak/filebrowser/files +cpu: 11th Gen Intel(R) Core(TM) i5-11320H @ 3.20GHz +BenchmarkFillIndex-8 10 3295862 ns/op 230448 B/op 1927 allocs/op +BenchmarkSearchAllIndexes-8 10 30033386 ns/op 19647893 B/op 298702 allocs/op +PASS +ok github.com/gtsteffaniak/filebrowser/files 0.392s PASS ok github.com/gtsteffaniak/filebrowser/fileutils 0.003s -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 h: 401 -2023/10/18 10:19:52 h: 401 -2023/10/18 10:19:52 h: 401 -2023/10/18 10:19:52 h: 401 -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:52 Saving new user: username -2023/10/18 10:19:53 h: 401 -2023/10/18 10:19:53 h: 401 +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 h: 401 +2023/11/24 13:57:20 h: 401 +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 Saving new user: username +2023/11/24 13:57:20 h: 401 +2023/11/24 13:57:20 h: 401 +2023/11/24 13:57:20 h: 401 +2023/11/24 13:57:20 h: 401 PASS ok github.com/gtsteffaniak/filebrowser/http 0.208s PASS -ok github.com/gtsteffaniak/filebrowser/img 0.124s -goos: linux -goarch: amd64 -pkg: github.com/gtsteffaniak/filebrowser/index -cpu: 11th Gen Intel(R) Core(TM) i5-11320H @ 3.20GHz -BenchmarkFillIndex-8 10 3239196 ns/op 11289 B/op 448 allocs/op -BenchmarkSearchAllIndexes-8 10 6645964 ns/op 3176834 B/op 59104 allocs/op -PASS -ok github.com/gtsteffaniak/filebrowser/index 0.126s +ok github.com/gtsteffaniak/filebrowser/img 0.118s PASS ok github.com/gtsteffaniak/filebrowser/rules 0.002s PASS diff --git a/backend/cmd/root.go b/backend/cmd/root.go index d9442d43..165e8abf 100644 --- a/backend/cmd/root.go +++ b/backend/cmd/root.go @@ -19,9 +19,9 @@ import ( "github.com/gtsteffaniak/filebrowser/auth" "github.com/gtsteffaniak/filebrowser/diskcache" + "github.com/gtsteffaniak/filebrowser/files" fbhttp "github.com/gtsteffaniak/filebrowser/http" "github.com/gtsteffaniak/filebrowser/img" - "github.com/gtsteffaniak/filebrowser/index" "github.com/gtsteffaniak/filebrowser/settings" "github.com/gtsteffaniak/filebrowser/users" ) @@ -47,7 +47,7 @@ func init() { var rootCmd = &cobra.Command{ Use: "filebrowser", Run: python(func(cmd *cobra.Command, args []string, d pythonData) { - serverConfig := settings.GlobalConfiguration.Server + serverConfig := settings.Config.Server if !d.hadDB { quickSetup(d) } @@ -64,7 +64,7 @@ var rootCmd = &cobra.Command{ fileCache = diskcache.New(afero.NewOsFs(), cacheDir) } // initialize indexing and schedule indexing ever n minutes (default 5) - go index.Initialize(serverConfig.IndexingInterval) + go files.InitializeIndex(serverConfig.IndexingInterval, true) _, err := os.Stat(serverConfig.Root) checkErr(err) var listener net.Listener @@ -118,23 +118,23 @@ func cleanupHandler(listener net.Listener, c chan os.Signal) { //nolint:interfac } func quickSetup(d pythonData) { - settings.GlobalConfiguration.Auth.Key = generateKey() - if settings.GlobalConfiguration.Auth.Method == "noauth" { + settings.Config.Auth.Key = generateKey() + if settings.Config.Auth.Method == "noauth" { err := d.store.Auth.Save(&auth.NoAuth{}) checkErr(err) } else { - settings.GlobalConfiguration.Auth.Method = "password" + settings.Config.Auth.Method = "password" err := d.store.Auth.Save(&auth.JSONAuth{}) checkErr(err) } - err := d.store.Settings.Save(&settings.GlobalConfiguration) + err := d.store.Settings.Save(&settings.Config) checkErr(err) - err = d.store.Settings.SaveServer(&settings.GlobalConfiguration.Server) + err = d.store.Settings.SaveServer(&settings.Config.Server) checkErr(err) user := &users.User{} - settings.GlobalConfiguration.UserDefaults.Apply(user) - user.Username = settings.GlobalConfiguration.Auth.AdminUsername - user.Password = settings.GlobalConfiguration.Auth.AdminPassword + settings.Config.UserDefaults.Apply(user) + user.Username = settings.Config.Auth.AdminUsername + user.Password = settings.Config.Auth.AdminPassword user.Perm.Admin = true user.Scope = "./" user.DarkMode = true diff --git a/backend/cmd/utils.go b/backend/cmd/utils.go index cae9854b..058134ff 100644 --- a/backend/cmd/utils.go +++ b/backend/cmd/utils.go @@ -83,7 +83,7 @@ func dbExists(path string) (bool, error) { func python(fn pythonFunc, cfg pythonConfig) cobraFunc { return func(cmd *cobra.Command, args []string) { data := pythonData{hadDB: true} - path := settings.GlobalConfiguration.Server.Database + path := settings.Config.Server.Database exists, err := dbExists(path) if err != nil { diff --git a/backend/filebrowser.yaml b/backend/filebrowser.yaml index 81040f2a..458ce2db 100644 --- a/backend/filebrowser.yaml +++ b/backend/filebrowser.yaml @@ -1,9 +1,10 @@ server: port: 8080 - baseURL: "/" + baseURL: "/" + root: "/srv" auth: method: password - signup: true + signup: false userDefaults: darkMode: true disableSettings: false diff --git a/backend/index/conditions.go b/backend/files/conditions.go similarity index 89% rename from backend/index/conditions.go rename to backend/files/conditions.go index ed7a3cae..54743be2 100644 --- a/backend/index/conditions.go +++ b/backend/files/conditions.go @@ -1,4 +1,4 @@ -package index +package files import ( "mime" @@ -15,22 +15,31 @@ var AllFiletypeOptions = []string{ "video", "doc", "dir", + "text", } var documentTypes = []string{ ".word", ".pdf", - ".txt", ".doc", ".docx", } - +var textTypes = []string{ + ".text", + ".sh", + ".yaml", + ".yml", + ".json", + ".bashrc", + ".zshrc", + ".env", +} var compressedFile = []string{ ".7z", ".rar", ".zip", ".tar", - ".tar.gz", - ".tar.xz", + ".gz", + ".xz", } type SearchOptions struct { @@ -137,12 +146,25 @@ func IsMatchingType(extension string, matchType string) bool { switch matchType { case "doc": return isDoc(extension) + case "pdf": + return extension == ".pdf" + case "text": + return isText(extension) case "archive": return isArchive(extension) } return false } +func isText(extension string) bool { + for _, typefile := range textTypes { + if extension == typefile { + return true + } + } + return false +} + func isDoc(extension string) bool { for _, typefile := range documentTypes { if extension == typefile { diff --git a/backend/files/file.go b/backend/files/file.go index ac1ca0fe..5a261ab2 100644 --- a/backend/files/file.go +++ b/backend/files/file.go @@ -8,33 +8,40 @@ import ( "encoding/hex" "hash" "io" - "log" "mime" "net/http" "os" - "path" - "path/filepath" + filepath "path/filepath" "strings" + "sync" "time" "github.com/spf13/afero" "github.com/gtsteffaniak/filebrowser/errors" "github.com/gtsteffaniak/filebrowser/rules" + "github.com/gtsteffaniak/filebrowser/users" +) + +var ( + bytesInMegabyte int64 = 1000000 + pathMutexes = make(map[string]*sync.Mutex) + pathMutexesMu sync.Mutex // Mutex to protect the pathMutexes map ) // FileInfo describes a file. type FileInfo struct { *Listing Fs afero.Fs `json:"-"` - Path string `json:"path"` + Path string `json:"path,omitempty"` Name string `json:"name"` Size int64 `json:"size"` - Extension string `json:"extension"` + Extension string `json:"-"` ModTime time.Time `json:"modified"` - Mode os.FileMode `json:"mode"` - IsDir bool `json:"isDir"` - IsSymlink bool `json:"isSymlink"` + CacheTime time.Time `json:"-"` + Mode os.FileMode `json:"-"` + IsDir bool `json:"isDir,omitempty"` + IsSymlink bool `json:"isSymlink,omitempty"` Type string `json:"type"` Subtitles []string `json:"subtitles,omitempty"` Content string `json:"content,omitempty"` @@ -54,6 +61,22 @@ type FileOptions struct { Content bool } +// Sorting constants +const ( + SortingByName = "name" + SortingBySize = "size" + SortingByModified = "modified" +) + +// Listing is a collection of files. +type Listing struct { + Items []*FileInfo `json:"items"` + Path string `json:"path"` + NumDirs int `json:"numDirs"` + NumFiles int `json:"numFiles"` + Sorting users.Sorting `json:"sorting"` +} + // NewFileInfo creates a File object from a path and a given user. This File // object will be automatically filled depending on if it is a directory // or a file. If it's a video file, it will also detect any subtitles. @@ -61,37 +84,126 @@ func NewFileInfo(opts FileOptions) (*FileInfo, error) { if !opts.Checker.Check(opts.Path) { return nil, os.ErrPermission } - - file, err := stat(opts) + file, err := stat(opts.Path, opts) // Pass opts.Path here if err != nil { return nil, err } - if opts.Expand { if file.IsDir { - if err := file.readListing(opts.Checker, opts.ReadHeader); err != nil { //nolint:govet + if err := file.readListing(opts.Path, opts.Checker, opts.ReadHeader); err != nil { //nolint:govet return nil, err } return file, nil } - - err = file.detectType(opts.Modify, opts.Content, true) + err = file.detectType(opts.Path, opts.Modify, opts.Content, true) if err != nil { return nil, err } } - return file, err } -func stat(opts FileOptions) (*FileInfo, error) { - var file *FileInfo +func FileInfoFaster(opts FileOptions) (*FileInfo, error) { + // Lock access for the specific path + pathMutex := getMutex(opts.Path) + pathMutex.Lock() + defer pathMutex.Unlock() + if !opts.Checker.Check(opts.Path) { + return nil, os.ErrPermission + } + index := GetIndex(rootPath) + trimmed := strings.TrimPrefix(opts.Path, "/") + if trimmed == "" { + trimmed = "/" + } + adjustedPath := makeIndexPath(trimmed, index.Root) + var info FileInfo + info, exists := index.GetMetadataInfo(adjustedPath) + if exists { + // Check if the cache time is less than 1 second + if time.Since(info.CacheTime) > time.Second { + go refreshFileInfo(opts) + } + // refresh cache after + return &info, nil + } else { + updated := refreshFileInfo(opts) + if !updated { + file, err := NewFileInfo(opts) + return file, err + } + info, exists = index.GetMetadataInfo(adjustedPath) + if !exists || info.Name == "" { + return &FileInfo{}, errors.ErrEmptyKey + } + return &info, nil + } +} +func refreshFileInfo(opts FileOptions) bool { + if !opts.Checker.Check(opts.Path) { + return false + } + file, err := stat(opts.Path, opts) // Pass opts.Path here + if err != nil { + return false + } + + index := GetIndex(rootPath) + trimmed := strings.TrimPrefix(opts.Path, "/") + if trimmed == "" { + trimmed = "/" + } + adjustedPath := makeIndexPath(trimmed, index.Root) + if file.IsDir { + err := file.readListing(opts.Path, opts.Checker, opts.ReadHeader) + if err != nil { + return false + } + //_, exists := index.GetFileMetadata(adjustedPath) + return index.UpdateFileMetadata(adjustedPath, *file) + } else { + //_, exists := index.GetFileMetadata(adjustedPath) + return index.UpdateFileMetadata(adjustedPath, *file) + } +} + +func stat(path string, opts FileOptions) (*FileInfo, error) { + var file *FileInfo if lstaterFs, ok := opts.Fs.(afero.Lstater); ok { - info, _, err := lstaterFs.LstatIfPossible(opts.Path) + info, _, err := lstaterFs.LstatIfPossible(path) + if err == nil { + file = &FileInfo{ + Fs: opts.Fs, + Path: opts.Path, + Name: info.Name(), + ModTime: info.ModTime(), + Mode: info.Mode(), + Size: info.Size(), + Extension: filepath.Ext(info.Name()), + Token: opts.Token, + } + if info.IsDir() { + file.IsDir = true + } + if info.Mode()&os.ModeSymlink != 0 { + file.IsSymlink = true + } + } + } + + if file == nil || file.IsSymlink { + info, err := opts.Fs.Stat(opts.Path) if err != nil { return nil, err } + + if file != nil && file.IsSymlink { + file.Size = info.Size() + file.IsDir = info.IsDir() + return file, nil + } + file = &FileInfo{ Fs: opts.Fs, Path: opts.Path, @@ -99,47 +211,12 @@ func stat(opts FileOptions) (*FileInfo, error) { ModTime: info.ModTime(), Mode: info.Mode(), IsDir: info.IsDir(), - IsSymlink: IsSymlink(info.Mode()), Size: info.Size(), Extension: filepath.Ext(info.Name()), Token: opts.Token, } } - // regular file - if file != nil && !file.IsSymlink { - return file, nil - } - - // fs doesn't support afero.Lstater interface or the file is a symlink - info, err := opts.Fs.Stat(opts.Path) - if err != nil { - // can't follow symlink - if file != nil && file.IsSymlink { - return file, nil - } - return nil, err - } - - // set correct file size in case of symlink - if file != nil && file.IsSymlink { - file.Size = info.Size() - file.IsDir = info.IsDir() - return file, nil - } - - file = &FileInfo{ - Fs: opts.Fs, - Path: opts.Path, - Name: info.Name(), - ModTime: info.ModTime(), - Mode: info.Mode(), - IsDir: info.IsDir(), - Size: info.Size(), - Extension: filepath.Ext(info.Name()), - Token: opts.Token, - } - return file, nil } @@ -160,19 +237,15 @@ func (i *FileInfo) Checksum(algo string) error { } defer reader.Close() - var h hash.Hash + hashFuncs := map[string]hash.Hash{ + "md5": md5.New(), + "sha1": sha1.New(), + "sha256": sha256.New(), + "sha512": sha512.New(), + } - //nolint:gosec - switch algo { - case "md5": - h = md5.New() - case "sha1": - h = sha1.New() - case "sha256": - h = sha256.New() - case "sha512": - h = sha512.New() - default: + h, ok := hashFuncs[algo] + if !ok { return errors.ErrInvalidOption } @@ -185,6 +258,7 @@ func (i *FileInfo) Checksum(algo string) error { return nil } +// RealPath gets the real path for the file, resolving symlinks if supported. func (i *FileInfo) RealPath() string { if realPathFs, ok := i.Fs.(interface { RealPath(name string) (fPath string, err error) @@ -198,72 +272,57 @@ func (i *FileInfo) RealPath() string { return i.Path } -// TODO: use constants -// -//nolint:goconst -func (i *FileInfo) detectType(modify, saveContent, readHeader bool) error { +// detectType detects the file type. +func (i *FileInfo) detectType(path string, modify, saveContent, readHeader bool) error { if IsNamedPipe(i.Mode) { i.Type = "blob" return nil } - // failing to detect the type should not return error. - // imagine the situation where a file in a dir with thousands - // of files couldn't be opened: we'd have immediately - // a 500 even though it doesn't matter. So we just log it. - - mimetype := mime.TypeByExtension(i.Extension) - var buffer []byte if readHeader { buffer = i.readFirstBytes() - + mimetype := mime.TypeByExtension(i.Extension) if mimetype == "" { - mimetype = http.DetectContentType(buffer) + http.DetectContentType(buffer) } } - - switch { - case strings.HasPrefix(mimetype, "video"): - i.Type = "video" - i.detectSubtitles() - return nil - case strings.HasPrefix(mimetype, "audio"): - i.Type = "audio" - return nil - case strings.HasPrefix(mimetype, "image"): - i.Type = "image" - return nil - case strings.HasSuffix(mimetype, "pdf"): - i.Type = "pdf" - return nil - case (strings.HasPrefix(mimetype, "text") || !isBinary(buffer)) && i.Size <= 10*1024*1024: // 10 MB - i.Type = "text" - - if !modify { - i.Type = "textImmutable" + ext := filepath.Ext(i.Name) + for _, fileType := range AllFiletypeOptions { + if IsMatchingType(ext, fileType) { + i.Type = fileType } - - if saveContent { - afs := &afero.Afero{Fs: i.Fs} - content, err := afs.ReadFile(i.Path) - if err != nil { - return err + switch i.Type { + case "text": + if !modify { + i.Type = "textImmutable" + } + if saveContent { + afs := &afero.Afero{Fs: i.Fs} + content, err := afs.ReadFile(path) + if err != nil { + return err + } + i.Content = string(content) + } + case "video": + parentDir := strings.TrimRight(path, i.Name) + i.detectSubtitles(parentDir) + case "doc": + if ext == ".pdf" { + i.Type = "pdf" } - - i.Content = string(content) } - return nil - default: + } + if i.Type == "" { i.Type = "blob" } - return nil } +// readFirstBytes reads the first bytes of the file. func (i *FileInfo) readFirstBytes() []byte { reader, err := i.Fs.Open(i.Path) if err != nil { - log.Print(err) i.Type = "blob" return nil } @@ -272,7 +331,6 @@ func (i *FileInfo) readFirstBytes() []byte { buffer := make([]byte, 512) //nolint:gomnd n, err := reader.Read(buffer) if err != nil && err != io.EOF { - log.Print(err) i.Type = "blob" return nil } @@ -280,29 +338,43 @@ func (i *FileInfo) readFirstBytes() []byte { return buffer[:n] } -func (i *FileInfo) detectSubtitles() { +// detectSubtitles detects subtitles for video files. +func (i *FileInfo) detectSubtitles(parentDir string) { if i.Type != "video" { return } - i.Subtitles = []string{} - ext := filepath.Ext(i.Path) + ext := filepath.Ext(i.Name) + dir, err := os.Open(parentDir) + if err != nil { + // Directory must have been deleted, remove it from the index + return + } + // Read the directory contents + files, err := dir.Readdir(-1) + if err != nil { + return + } - // detect multiple languages. Base*.vtt - // TODO: give subtitles descriptive names (lang) and track attributes - parentDir := strings.TrimRight(i.Path, i.Name) - dir, err := afero.ReadDir(i.Fs, parentDir) - if err == nil { - base := strings.TrimSuffix(i.Name, ext) - for _, f := range dir { - if !f.IsDir() && strings.HasPrefix(f.Name(), base) && strings.HasSuffix(f.Name(), ".vtt") { - i.Subtitles = append(i.Subtitles, path.Join(parentDir, f.Name())) + base := strings.TrimSuffix(i.Name, ext) + subtitleExts := []string{".vtt", ".txt", ".srt", ".lrc"} + + for _, f := range files { + if f.IsDir() || !strings.HasPrefix(f.Name(), base) { + continue + } + + for _, subtitleExt := range subtitleExts { + if strings.HasSuffix(f.Name(), subtitleExt) { + i.Subtitles = append(i.Subtitles, filepath.Join(parentDir, f.Name())) + break } } } } -func (i *FileInfo) readListing(checker rules.Checker, readHeader bool) error { +// readListing reads the contents of a directory and fills the listing. +func (i *FileInfo) readListing(path string, checker rules.Checker, readHeader bool) error { afs := &afero.Afero{Fs: i.Fs} dir, err := afs.ReadDir(i.Path) if err != nil { @@ -311,13 +383,14 @@ func (i *FileInfo) readListing(checker rules.Checker, readHeader bool) error { listing := &Listing{ Items: []*FileInfo{}, + Path: i.Path, NumDirs: 0, NumFiles: 0, } for _, f := range dir { name := f.Name() - fPath := path.Join(i.Path, name) + fPath := filepath.Join(i.Path, name) if !checker.Check(fPath) { continue @@ -326,8 +399,6 @@ func (i *FileInfo) readListing(checker rules.Checker, readHeader bool) error { isSymlink, isInvalidLink := false, false if IsSymlink(f.Mode()) { isSymlink = true - // It's a symbolic link. We try to follow it. If it doesn't work, - // we stay with the link information instead of the target's. info, err := i.Fs.Stat(fPath) if err == nil { f = info @@ -337,15 +408,16 @@ func (i *FileInfo) readListing(checker rules.Checker, readHeader bool) error { } file := &FileInfo{ - Fs: i.Fs, - Name: name, - Size: f.Size(), - ModTime: f.ModTime(), - Mode: f.Mode(), - IsDir: f.IsDir(), - IsSymlink: isSymlink, - Extension: filepath.Ext(name), - Path: fPath, + Name: name, + Size: f.Size(), + ModTime: f.ModTime(), + Mode: f.Mode(), + } + if f.IsDir() { + file.IsDir = true + } + if isSymlink { + file.IsSymlink = true } if file.IsDir { @@ -356,7 +428,7 @@ func (i *FileInfo) readListing(checker rules.Checker, readHeader bool) error { if isInvalidLink { file.Type = "invalid_link" } else { - err := file.detectType(true, false, readHeader) + err := file.detectType(path, true, false, readHeader) if err != nil { return err } @@ -369,3 +441,23 @@ func (i *FileInfo) readListing(checker rules.Checker, readHeader bool) error { i.Listing = listing return nil } +func IsNamedPipe(mode os.FileMode) bool { + return mode&os.ModeNamedPipe != 0 +} + +func IsSymlink(mode os.FileMode) bool { + return mode&os.ModeSymlink != 0 +} + +func getMutex(path string) *sync.Mutex { + // Lock access to pathMutexes map + pathMutexesMu.Lock() + defer pathMutexesMu.Unlock() + + // Create a mutex for the path if it doesn't exist + if pathMutexes[path] == nil { + pathMutexes[path] = &sync.Mutex{} + } + + return pathMutexes[path] +} diff --git a/backend/files/indexing.go b/backend/files/indexing.go new file mode 100644 index 00000000..24293eff --- /dev/null +++ b/backend/files/indexing.go @@ -0,0 +1,165 @@ +package files + +import ( + "bytes" + "log" + "os" + "strings" + "sync" + "time" + + "github.com/gtsteffaniak/filebrowser/settings" +) + +type Directory struct { + Metadata map[string]FileInfo + Files string +} +type File struct { + Name string + IsDir bool +} + +type Index struct { + Root string + Directories map[string]Directory + NumDirs int + NumFiles int + inProgress bool + quickList []File + LastIndexed time.Time + mu sync.RWMutex +} + +var ( + rootPath string = "/srv" + indexes []*Index + indexesMutex sync.RWMutex +) + +func InitializeIndex(intervalMinutes uint32, schedule bool) { + if schedule { + go indexingScheduler(intervalMinutes) + } +} + +func indexingScheduler(intervalMinutes uint32) { + if settings.Config.Server.Root != "" { + rootPath = settings.Config.Server.Root + } + si := GetIndex(rootPath) + log.Printf("Indexing Files...") + log.Printf("Configured to run every %v minutes", intervalMinutes) + log.Printf("Indexing from root: %s", si.Root) + for { + startTime := time.Now() + // Set the indexing flag to indicate that indexing is in progress + si.resetCount() + // Perform the indexing operation + err := si.indexFiles(si.Root) + si.quickList = []File{} + // Reset the indexing flag to indicate that indexing has finished + si.inProgress = false + // Update the LastIndexed time + si.LastIndexed = time.Now() + if err != nil { + log.Printf("Error during indexing: %v", err) + } + if si.NumFiles+si.NumDirs > 0 { + timeIndexedInSeconds := int(time.Since(startTime).Seconds()) + log.Println("Successfully indexed files.") + log.Printf("Time spent indexing: %v seconds\n", timeIndexedInSeconds) + log.Printf("Files found: %v\n", si.NumFiles) + log.Printf("Directories found: %v\n", si.NumDirs) + } + // Sleep for the specified interval + time.Sleep(time.Duration(intervalMinutes) * time.Minute) + } +} + +// Define a function to recursively index files and directories +func (si *Index) indexFiles(path string) error { + // Check if the current directory has been modified since the last indexing + path = strings.TrimSuffix(path, "/") + adjustedPath := makeIndexPath(path, si.Root) + dir, err := os.Open(path) + if err != nil { + // Directory must have been deleted, remove it from the index + si.RemoveDirectory(adjustedPath) + } + dirInfo, err := dir.Stat() + if err != nil { + dir.Close() + return err + } + + // Compare the last modified time of the directory with the last indexed time + lastIndexed := si.LastIndexed + if dirInfo.ModTime().Before(lastIndexed) { + dir.Close() + return nil + } + + // Read the directory contents + files, err := dir.Readdir(-1) + if err != nil { + return err + } + dir.Close() + si.UpdateQuickList(files) + si.InsertFiles(path) + // done separately for memory efficiency on recursion + si.InsertDirs(path) + return nil +} + +func (si *Index) InsertFiles(path string) { + adjustedPath := makeIndexPath(path, si.Root) + subDirectory := Directory{} + buffer := bytes.Buffer{} + + for _, f := range si.GetQuickList() { + buffer.WriteString(f.Name + ";") + si.UpdateCount("files") + } + // Use GetMetadataInfo and SetFileMetadata for safer read and write operations + subDirectory.Files = buffer.String() + si.SetDirectoryInfo(adjustedPath, subDirectory) +} + +func (si *Index) InsertDirs(path string) { + adjustedPath := makeIndexPath(path, si.Root) + for _, f := range si.GetQuickList() { + if f.IsDir { + if _, exists := si.Directories[adjustedPath]; exists { + si.UpdateCount("dirs") + // Add or update the directory in the map + if adjustedPath == "/" { + si.SetDirectoryInfo("/"+f.Name, Directory{}) + } else { + si.SetDirectoryInfo(adjustedPath+"/"+f.Name, Directory{}) + } + } + err := si.indexFiles(path + "/" + f.Name) + if err != nil { + if err.Error() == "invalid argument" { + log.Printf("Could not index \"%v\": %v \n", path, "Permission Denied") + } else { + log.Printf("Could not index \"%v\": %v \n", path, err) + } + } + } + } +} + +func makeIndexPath(path string, root string) string { + if path == root { + return "/" + } + adjustedPath := strings.TrimPrefix(path, root+"/") + adjustedPath = strings.TrimSuffix(adjustedPath, "/") + if adjustedPath == "" { + adjustedPath = "/" + } + return adjustedPath +} diff --git a/backend/index/indexing_test.go b/backend/files/indexing_test.go similarity index 61% rename from backend/index/indexing_test.go rename to backend/files/indexing_test.go index 01f4e7eb..890679a2 100644 --- a/backend/index/indexing_test.go +++ b/backend/files/indexing_test.go @@ -1,4 +1,4 @@ -package index +package files import ( "encoding/json" @@ -6,28 +6,35 @@ import ( "reflect" "testing" "time" + + "github.com/gtsteffaniak/filebrowser/settings" ) func BenchmarkFillIndex(b *testing.B) { - indexes = Index{ - Dirs: []string{}, - Files: []string{}, - } + InitializeIndex(5, false) + si := GetIndex(settings.Config.Server.Root) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - createMockData(50, 3) // 1000 dirs, 3 files per dir + si.createMockData(50, 3) // 1000 dirs, 3 files per dir } } -func createMockData(numDirs, numFilesPerDir int) { +func (si *Index) createMockData(numDirs, numFilesPerDir int) { for i := 0; i < numDirs; i++ { dirName := generateRandomPath(rand.Intn(3) + 1) - addToIndex("/", dirName, true) + files := []File{} + // Append a new Directory to the slice for j := 0; j < numFilesPerDir; j++ { - fileName := "file-" + getRandomTerm() + getRandomExtension() - addToIndex("/"+dirName, fileName, false) + newFile := File{ + Name: "file-" + getRandomTerm() + getRandomExtension(), + IsDir: false, + } + files = append(files, newFile) } + si.UpdateQuickListForTests(files) + si.InsertFiles(dirName) + si.InsertDirs(dirName) } } @@ -91,7 +98,7 @@ func TestGetIndex(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := GetIndex(); !reflect.DeepEqual(got, tt.want) { + if got := GetIndex("root"); !reflect.DeepEqual(got, tt.want) { t.Errorf("GetIndex() = %v, want %v", got, tt.want) } }) @@ -110,7 +117,7 @@ func TestInitializeIndex(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - Initialize(tt.args.intervalMinutes) + InitializeIndex(tt.args.intervalMinutes, false) }) } } @@ -131,54 +138,3 @@ func Test_indexingScheduler(t *testing.T) { }) } } - -func Test_indexFiles(t *testing.T) { - type args struct { - path string - numFiles *int - numDirs *int - } - tests := []struct { - name string - args args - want int - want1 int - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1, err := indexFiles(tt.args.path, tt.args.numFiles, tt.args.numDirs) - if (err != nil) != tt.wantErr { - t.Errorf("indexFiles() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("indexFiles() got = %v, want %v", got, tt.want) - } - if got1 != tt.want1 { - t.Errorf("indexFiles() got1 = %v, want %v", got1, tt.want1) - } - }) - } -} - -func Test_addToIndex(t *testing.T) { - type args struct { - path string - fileName string - isDir bool - } - tests := []struct { - name string - args args - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - addToIndex(tt.args.path, tt.args.fileName, tt.args.isDir) - }) - } -} diff --git a/backend/files/listing.go b/backend/files/listing.go deleted file mode 100644 index f35ad8cb..00000000 --- a/backend/files/listing.go +++ /dev/null @@ -1,111 +0,0 @@ -package files - -import ( - "sort" - "strings" - - "github.com/maruel/natural" -) - -// Listing is a collection of files. -type Listing struct { - Items []*FileInfo `json:"items"` - NumDirs int `json:"numDirs"` - NumFiles int `json:"numFiles"` - Sorting Sorting `json:"sorting"` -} - -// ApplySort applies the sort order using .Order and .Sort -// -//nolint:goconst -func (l Listing) ApplySort() { - // Check '.Order' to know how to sort - // TODO: use enum - if !l.Sorting.Asc { - switch l.Sorting.By { - case "name": - sort.Sort(sort.Reverse(byName(l))) - case "size": - sort.Sort(sort.Reverse(bySize(l))) - case "modified": - sort.Sort(sort.Reverse(byModified(l))) - default: - // If not one of the above, do nothing - return - } - } else { // If we had more Orderings we could add them here - switch l.Sorting.By { - case "name": - sort.Sort(byName(l)) - case "size": - sort.Sort(bySize(l)) - case "modified": - sort.Sort(byModified(l)) - default: - sort.Sort(byName(l)) - return - } - } -} - -// Implement sorting for Listing -type byName Listing -type bySize Listing -type byModified Listing - -// By Name -func (l byName) Len() int { - return len(l.Items) -} - -func (l byName) Swap(i, j int) { - l.Items[i], l.Items[j] = l.Items[j], l.Items[i] -} - -// Treat upper and lower case equally -func (l byName) Less(i, j int) bool { - if l.Items[i].IsDir && !l.Items[j].IsDir { - return l.Sorting.Asc - } - - if !l.Items[i].IsDir && l.Items[j].IsDir { - return !l.Sorting.Asc - } - - return natural.Less(strings.ToLower(l.Items[j].Name), strings.ToLower(l.Items[i].Name)) -} - -// By Size -func (l bySize) Len() int { - return len(l.Items) -} - -func (l bySize) Swap(i, j int) { - l.Items[i], l.Items[j] = l.Items[j], l.Items[i] -} - -const directoryOffset = -1 << 31 // = math.MinInt32 -func (l bySize) Less(i, j int) bool { - iSize, jSize := l.Items[i].Size, l.Items[j].Size - if l.Items[i].IsDir { - iSize = directoryOffset + iSize - } - if l.Items[j].IsDir { - jSize = directoryOffset + jSize - } - return iSize < jSize -} - -// By Modified -func (l byModified) Len() int { - return len(l.Items) -} - -func (l byModified) Swap(i, j int) { - l.Items[i], l.Items[j] = l.Items[j], l.Items[i] -} - -func (l byModified) Less(i, j int) bool { - iModified, jModified := l.Items[i].ModTime, l.Items[j].ModTime - return iModified.Sub(jModified) < 0 -} diff --git a/backend/index/search_index.go b/backend/files/search.go similarity index 79% rename from backend/index/search_index.go rename to backend/files/search.go index 92e1b68f..8f76d194 100644 --- a/backend/index/search_index.go +++ b/backend/files/search.go @@ -1,4 +1,4 @@ -package index +package files import ( "math/rand" @@ -12,9 +12,7 @@ import ( var ( sessionInProgress sync.Map - mutex sync.RWMutex - maxSearchResults = 100 - bytesInMegabyte int64 = 1000000 + maxSearchResults = 100 ) func (si *Index) Search(search string, scope string, sourceSession string) ([]string, map[string]map[string]bool) { @@ -24,46 +22,61 @@ func (si *Index) Search(search string, scope string, sourceSession string) ([]st runningHash := generateRandomHash(4) sessionInProgress.Store(sourceSession, runningHash) // Store the value in the sync.Map searchOptions := ParseSearch(search) - mutex.RLock() - defer mutex.RUnlock() fileListTypes := make(map[string]map[string]bool) matching := []string{} + count := 0 + for _, searchTerm := range searchOptions.Terms { if searchTerm == "" { continue } - // Iterate over the embedded index.Index fields Dirs and Files - for _, i := range []string{"Dirs", "Files"} { - isDir := false - count := 0 - var paths []string - - switch i { - case "Dirs": - isDir = true - paths = si.Dirs - case "Files": - paths = si.Files + si.mu.Lock() + defer si.mu.Unlock() + for dirName, dir := range si.Directories { + isDir := true + files := strings.Split(dir.Files, ";") + value, found := sessionInProgress.Load(sourceSession) + if !found || value != runningHash { + return []string{}, map[string]map[string]bool{} } - for _, path := range paths { + if count > maxSearchResults { + break + } + pathName := scopedPathNameFilter(dirName, scope, isDir) + if pathName == "" { + continue // path not matched + } + + fileTypes := map[string]bool{} + matches, fileType := containsSearchTerm(dirName, searchTerm, *searchOptions, isDir, fileTypes) + if matches { + fileListTypes[pathName] = fileType + matching = append(matching, pathName) + count++ + } + isDir = false + for _, file := range files { + if file == "" { + continue + } value, found := sessionInProgress.Load(sourceSession) if !found || value != runningHash { return []string{}, map[string]map[string]bool{} } + if count > maxSearchResults { break } - pathName := scopedPathNameFilter(path, scope, isDir) - if pathName == "" { - continue - } + fullName := pathName + file fileTypes := map[string]bool{} - matches, fileType := containsSearchTerm(path, searchTerm, *searchOptions, isDir, fileTypes) + + matches, fileType := containsSearchTerm(fullName, searchTerm, *searchOptions, isDir, fileTypes) if !matches { continue } - fileListTypes[pathName] = fileType - matching = append(matching, pathName) + + fileListTypes[fullName] = fileType + matching = append(matching, fullName) count++ } } diff --git a/backend/index/search_index_test.go b/backend/files/search_test.go similarity index 73% rename from backend/index/search_index_test.go rename to backend/files/search_test.go index ac0ab407..503efd35 100644 --- a/backend/index/search_index_test.go +++ b/backend/files/search_test.go @@ -1,4 +1,4 @@ -package index +package files import ( "reflect" @@ -8,12 +8,10 @@ import ( ) func BenchmarkSearchAllIndexes(b *testing.B) { - indexes = Index{ - Dirs: []string{}, - Files: []string{}, - } - // Create mock data - createMockData(50, 3) // 1000 dirs, 3 files per dir + InitializeIndex(5, false) + si := GetIndex(rootPath) + + si.createMockData(50, 3) // 1000 dirs, 3 files per dir // Generate 100 random search terms searchTerms := generateRandomSearchTerms(100) @@ -23,7 +21,7 @@ func BenchmarkSearchAllIndexes(b *testing.B) { for i := 0; i < b.N; i++ { // Execute the SearchAllIndexes function for _, term := range searchTerms { - indexes.Search(term, "/", "test") + si.Search(term, "/", "test") } } } @@ -76,20 +74,37 @@ func TestParseSearch(t *testing.T) { } } +func TestSearchWhileIndexing(t *testing.T) { + InitializeIndex(5, false) + si := GetIndex(rootPath) + // Generate 100 random search terms + // Generate 100 random search terms + searchTerms := generateRandomSearchTerms(10) + for i := 0; i < 5; i++ { + // Execute the SearchAllIndexes function + go si.createMockData(100, 100) // 1000 dirs, 3 files per dir + for _, term := range searchTerms { + go si.Search(term, "/", "test") + } + } +} + func TestSearchIndexes(t *testing.T) { - indexes = Index{ - Dirs: []string{ - "/test", - "/test/path", - "/new/test/path", - }, - Files: []string{ - "/test/path/file.txt", - "/test/audio1.wav", - "/new/test/audio.wav", - "/new/test/video.mp4", - "/new/test/video.MP4", - "/new/test/path/archive.zip", + index := Index{ + Directories: map[string]Directory{ + "test": { + Files: "audio1.wav;", + }, + "test/path": { + Files: "file.txt;", + }, + "new": {}, + "new/test": { + Files: "audio.wav;video.mp4;video.MP4;", + }, + "new/test/path": { + Files: "archive.zip;", + }, }, } tests := []struct { @@ -109,9 +124,10 @@ func TestSearchIndexes(t *testing.T) { { search: "test", scope: "/", - expectedResult: []string{"test/"}, + expectedResult: []string{"test/", "new/test/"}, expectedTypes: map[string]map[string]bool{ - "test/": map[string]bool{"dir": true}, + "test/": map[string]bool{"dir": true}, + "new/test/": map[string]bool{"dir": true}, }, }, { @@ -137,19 +153,35 @@ func TestSearchIndexes(t *testing.T) { } for _, tt := range tests { t.Run(tt.search, func(t *testing.T) { - actualResult, actualTypes := indexes.Search(tt.search, tt.scope, "") + actualResult, actualTypes := index.Search(tt.search, tt.scope, "") assert.Equal(t, tt.expectedResult, actualResult) - if len(tt.expectedTypes) > 0 { - for key, value := range tt.expectedTypes { - actualValue, exists := actualTypes[key] - assert.True(t, exists, "Expected type key '%s' not found in actual types", key) - assert.Equal(t, value, actualValue, "Type value mismatch for key '%s'", key) - } + if !reflect.DeepEqual(tt.expectedTypes, actualTypes) { + t.Fatalf("\n got: %+v\n want: %+v", actualTypes, tt.expectedTypes) } }) } } +func Test_scopedPathNameFilter(t *testing.T) { + type args struct { + pathName string + scope string + } + tests := []struct { + name string + args args + want string + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := scopedPathNameFilter(tt.args.pathName, tt.args.scope, false); got != tt.want { + t.Errorf("scopedPathNameFilter() = %v, want %v", got, tt.want) + } + }) + } +} func Test_isDoc(t *testing.T) { type args struct { extension string diff --git a/backend/files/sorting.go b/backend/files/sorting.go deleted file mode 100644 index ecdc3df6..00000000 --- a/backend/files/sorting.go +++ /dev/null @@ -1,7 +0,0 @@ -package files - -// Sorting contains a sorting order. -type Sorting struct { - By string `json:"by"` - Asc bool `json:"asc"` -} diff --git a/backend/files/sync.go b/backend/files/sync.go new file mode 100644 index 00000000..99986f54 --- /dev/null +++ b/backend/files/sync.go @@ -0,0 +1,171 @@ +package files + +import ( + "io/fs" + "log" + "time" + + "github.com/gtsteffaniak/filebrowser/settings" +) + +// GetFileMetadata retrieves the FileInfo from the specified directory in the index. +func (si *Index) GetFileMetadata(adjustedPath string) (FileInfo, bool) { + si.mu.RLock() + dir, exists := si.Directories[adjustedPath] + si.mu.RUnlock() + if exists { + // Initialize the Metadata map if it is nil + if dir.Metadata == nil { + dir.Metadata = make(map[string]FileInfo) + si.SetDirectoryInfo(adjustedPath, dir) + return FileInfo{}, false + } else { + return dir.Metadata[adjustedPath], true + } + } + return FileInfo{}, false +} + +// UpdateFileMetadata updates the FileInfo for the specified directory in the index. +func (si *Index) UpdateFileMetadata(adjustedPath string, info FileInfo) bool { + si.mu.RLock() + dir, exists := si.Directories[adjustedPath] + si.mu.RUnlock() + if exists { + // Initialize the Metadata map if it is nil + if dir.Metadata == nil { + dir.Metadata = make(map[string]FileInfo) + } + // Release the read lock before calling SetFileMetadata + } + return si.SetFileMetadata(adjustedPath, info) +} + +// SetFileMetadata sets the FileInfo for the specified directory in the index. +func (si *Index) SetFileMetadata(adjustedPath string, info FileInfo) bool { + si.mu.Lock() + defer si.mu.Unlock() + _, exists := si.Directories[adjustedPath] + if !exists { + return false + } + info.CacheTime = time.Now() + si.Directories[adjustedPath].Metadata[adjustedPath] = info + return true +} + +// GetMetadataInfo retrieves the FileInfo from the specified directory in the index. +func (si *Index) GetMetadataInfo(adjustedPath string) (FileInfo, bool) { + si.mu.RLock() + dir, exists := si.Directories[adjustedPath] + si.mu.RUnlock() + if exists { + // Initialize the Metadata map if it is nil + if dir.Metadata == nil { + dir.Metadata = make(map[string]FileInfo) + si.SetDirectoryInfo(adjustedPath, dir) + } + info, metadataExists := dir.Metadata[adjustedPath] + return info, metadataExists + } + return FileInfo{}, false +} + +// SetDirectoryInfo sets the directory information in the index. +func (si *Index) SetDirectoryInfo(adjustedPath string, dir Directory) { + si.mu.Lock() + si.Directories[adjustedPath] = dir + si.mu.Unlock() +} + +// SetDirectoryInfo sets the directory information in the index. +func (si *Index) GetDirectoryInfo(adjustedPath string) (Directory, bool) { + si.mu.RLock() + dir, exists := si.Directories[adjustedPath] + si.mu.RUnlock() + if exists { + return dir, true + } + return Directory{}, false +} + +func (si *Index) RemoveDirectory(path string) { + si.mu.Lock() + defer si.mu.Unlock() + delete(si.Directories, path) +} + +func (si *Index) UpdateCount(given string) { + si.mu.Lock() + defer si.mu.Unlock() + if given == "files" { + si.NumFiles++ + } else if given == "dirs" { + si.NumDirs++ + } else { + log.Println("could not update unknown type: ", given) + } +} + +func (si *Index) resetCount() { + si.mu.Lock() + defer si.mu.Unlock() + si.NumDirs = 0 + si.NumFiles = 0 + si.inProgress = true +} + +func GetIndex(root string) *Index { + for _, index := range indexes { + if index.Root == root { + return index + } + } + if settings.Config.Server.Root != "" { + rootPath = settings.Config.Server.Root + } + newIndex := &Index{ + Root: rootPath, + Directories: make(map[string]Directory), // Initialize the map + NumDirs: 0, + NumFiles: 0, + inProgress: false, + } + indexesMutex.Lock() + indexes = append(indexes, newIndex) + indexesMutex.Unlock() + return newIndex +} + +func (si *Index) UpdateQuickList(files []fs.FileInfo) { + si.mu.Lock() + defer si.mu.Unlock() + si.quickList = []File{} + for _, file := range files { + newFile := File{ + Name: file.Name(), + IsDir: file.IsDir(), + } + si.quickList = append(si.quickList, newFile) + } +} + +func (si *Index) UpdateQuickListForTests(files []File) { + si.mu.Lock() + defer si.mu.Unlock() + si.quickList = []File{} + for _, file := range files { + newFile := File{ + Name: file.Name, + IsDir: file.IsDir, + } + si.quickList = append(si.quickList, newFile) + } +} + +func (si *Index) GetQuickList() []File { + si.mu.Lock() + defer si.mu.Unlock() + newQuickList := si.quickList + return newQuickList +} diff --git a/backend/files/utils.go b/backend/files/utils.go deleted file mode 100644 index f4b0365d..00000000 --- a/backend/files/utils.go +++ /dev/null @@ -1,59 +0,0 @@ -package files - -import ( - "os" - "unicode/utf8" -) - -func isBinary(content []byte) bool { - maybeStr := string(content) - runeCnt := utf8.RuneCount(content) - runeIndex := 0 - gotRuneErrCnt := 0 - firstRuneErrIndex := -1 - - const ( - // 8 and below are control chars (e.g. backspace, null, eof, etc) - maxControlCharsCode = 8 - // 0xFFFD(65533) is the "error" Rune or "Unicode replacement character" - // see https://golang.org/pkg/unicode/utf8/#pkg-constants - unicodeReplacementChar = 0xFFFD - ) - - for _, b := range maybeStr { - if b <= maxControlCharsCode { - return true - } - - if b == unicodeReplacementChar { - // if it is not the last (utf8.UTFMax - x) rune - if runeCnt > utf8.UTFMax && runeIndex < runeCnt-utf8.UTFMax { - return true - } - // else it is the last (utf8.UTFMax - x) rune - // there maybe Vxxx, VVxx, VVVx, thus, we may got max 3 0xFFFD rune (assume V is the byte we got) - // for Chinese, it can only be Vxx, VVx, we may got max 2 0xFFFD rune - gotRuneErrCnt++ - - // mark the first time - if firstRuneErrIndex == -1 { - firstRuneErrIndex = runeIndex - } - } - runeIndex++ - } - - // if last (utf8.UTFMax - x ) rune has the "error" Rune, but not all - if firstRuneErrIndex != -1 && gotRuneErrCnt != runeCnt-firstRuneErrIndex { - return true - } - return false -} - -func IsNamedPipe(mode os.FileMode) bool { - return mode&os.ModeNamedPipe != 0 -} - -func IsSymlink(mode os.FileMode) bool { - return mode&os.ModeSymlink != 0 -} diff --git a/backend/go.mod b/backend/go.mod index 83bc14f3..b3dd71d6 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -11,7 +11,6 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/go-cmp v0.5.9 github.com/gorilla/mux v1.8.0 - github.com/maruel/natural v1.1.0 github.com/marusama/semaphore/v2 v2.5.0 github.com/mholt/archiver/v3 v3.5.1 github.com/shirou/gopsutil/v3 v3.23.8 @@ -20,10 +19,9 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce - golang.org/x/crypto v0.12.0 + golang.org/x/crypto v0.14.0 golang.org/x/image v0.12.0 golang.org/x/text v0.13.0 - gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( @@ -50,8 +48,8 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.4 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.11.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/backend/go.sum b/backend/go.sum index 525f9e3f..91951de2 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -204,8 +204,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= -github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= github.com/marusama/semaphore/v2 v2.5.0 h1:o/1QJD9DBYOWRnDhPwDVAXQn6mQYD0gZaS1Tpx6DJGM= github.com/marusama/semaphore/v2 v2.5.0/go.mod h1:z9nMiNUekt/LTpTUQdpp+4sJeYqUGpwMHfW0Z8V8fnQ= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= @@ -282,8 +280,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -359,8 +357,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -427,8 +425,9 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -595,8 +594,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/backend/http/auth.go b/backend/http/auth.go index 612efca6..2d1327c7 100644 --- a/backend/http/auth.go +++ b/backend/http/auth.go @@ -112,7 +112,7 @@ type signupBody struct { } var signupHandler = func(w http.ResponseWriter, r *http.Request, d *data) (int, error) { - if !settings.GlobalConfiguration.Auth.Signup { + if !settings.Config.Auth.Signup { return http.StatusMethodNotAllowed, nil } @@ -134,7 +134,7 @@ var signupHandler = func(w http.ResponseWriter, r *http.Request, d *data) (int, Username: info.Username, Password: info.Password, } - settings.GlobalConfiguration.UserDefaults.Apply(user) + settings.Config.UserDefaults.Apply(user) userHome, err := d.settings.MakeUserDir(user.Username, user.Scope, d.server.Root) if err != nil { log.Printf("create user: failed to mkdir user home dir: [%s]", userHome) @@ -142,7 +142,7 @@ var signupHandler = func(w http.ResponseWriter, r *http.Request, d *data) (int, } user.Scope = userHome log.Printf("new user: %s, home dir: [%s].", user.Username, userHome) - settings.GlobalConfiguration.UserDefaults.Apply(user) + settings.Config.UserDefaults.Apply(user) err = d.store.Users.Save(user) if err == errors.ErrExist { return http.StatusConflict, err diff --git a/backend/http/preview.go b/backend/http/preview.go index c7f87ca0..57d68f51 100644 --- a/backend/http/preview.go +++ b/backend/http/preview.go @@ -45,7 +45,7 @@ func previewHandler(imgSvc ImgService, fileCache FileCache, enableThumbnails, re return http.StatusBadRequest, err } - file, err := files.NewFileInfo(files.FileOptions{ + file, err := files.FileInfoFaster(files.FileOptions{ Fs: d.user.Fs, Path: "/" + vars["path"], Modify: d.user.Perm.Modify, @@ -53,10 +53,10 @@ func previewHandler(imgSvc ImgService, fileCache FileCache, enableThumbnails, re ReadHeader: d.server.TypeDetectionByHeader, Checker: d, }) + if err != nil { return errToStatus(err), err } - setContentDisposition(w, r, file) switch file.Type { @@ -81,7 +81,6 @@ func handleImagePreview( (previewSize == PreviewSizeThumb && !enableThumbnails) { return rawFileHandler(w, r, file) } - format, err := imgSvc.FormatFromExtension(file.Extension) // Unsupported extensions directly return the raw data if err == img.ErrUnsupportedFormat || format == img.FormatGif { diff --git a/backend/http/public.go b/backend/http/public.go index 890ded35..8ba87ab6 100644 --- a/backend/http/public.go +++ b/backend/http/public.go @@ -13,6 +13,7 @@ import ( "github.com/gtsteffaniak/filebrowser/files" "github.com/gtsteffaniak/filebrowser/share" + "github.com/gtsteffaniak/filebrowser/users" ) var withHashFile = func(fn handleFunc) handleFunc { @@ -35,7 +36,7 @@ var withHashFile = func(fn handleFunc) handleFunc { d.user = user - file, err := files.NewFileInfo(files.FileOptions{ + file, err := files.FileInfoFaster(files.FileOptions{ Fs: d.user.Fs, Path: link.Path, Modify: d.user.Perm.Modify, @@ -62,7 +63,7 @@ var withHashFile = func(fn handleFunc) handleFunc { // set fs root to the shared file/folder d.user.Fs = afero.NewBasePathFs(d.user.Fs, basePath) - file, err = files.NewFileInfo(files.FileOptions{ + file, err = files.FileInfoFaster(files.FileOptions{ Fs: d.user.Fs, Path: filePath, Modify: d.user.Perm.Modify, @@ -98,8 +99,7 @@ var publicShareHandler = withHashFile(func(w http.ResponseWriter, r *http.Reques file := d.raw.(*files.FileInfo) if file.IsDir { - file.Listing.Sorting = files.Sorting{By: "name", Asc: false} - file.Listing.ApplySort() + file.Listing.Sorting = users.Sorting{By: "name", Asc: false} return renderJSON(w, r, file) } diff --git a/backend/http/raw.go b/backend/http/raw.go index 41efa4a5..dc72a437 100644 --- a/backend/http/raw.go +++ b/backend/http/raw.go @@ -81,7 +81,7 @@ var rawHandler = withUser(func(w http.ResponseWriter, r *http.Request, d *data) return http.StatusAccepted, nil } - file, err := files.NewFileInfo(files.FileOptions{ + file, err := files.FileInfoFaster(files.FileOptions{ Fs: d.user.Fs, Path: r.URL.Path, Modify: d.user.Perm.Modify, diff --git a/backend/http/resource.go b/backend/http/resource.go index 2dc43ffc..af375ba1 100644 --- a/backend/http/resource.go +++ b/backend/http/resource.go @@ -20,7 +20,7 @@ import ( ) var resourceGetHandler = withUser(func(w http.ResponseWriter, r *http.Request, d *data) (int, error) { - file, err := files.NewFileInfo(files.FileOptions{ + file, err := files.FileInfoFaster(files.FileOptions{ Fs: d.user.Fs, Path: r.URL.Path, Modify: d.user.Perm.Modify, @@ -32,13 +32,10 @@ var resourceGetHandler = withUser(func(w http.ResponseWriter, r *http.Request, d if err != nil { return errToStatus(err), err } - if file.IsDir { file.Listing.Sorting = d.user.Sorting - file.Listing.ApplySort() return renderJSON(w, r, file) } - if checksum := r.URL.Query().Get("checksum"); checksum != "" { err := file.Checksum(checksum) if err == errors.ErrInvalidOption { @@ -46,9 +43,6 @@ var resourceGetHandler = withUser(func(w http.ResponseWriter, r *http.Request, d } else if err != nil { return http.StatusInternalServerError, err } - - // do not waste bandwidth if we just want the checksum - file.Content = "" } return renderJSON(w, r, file) @@ -60,7 +54,7 @@ func resourceDeleteHandler(fileCache FileCache) handleFunc { return http.StatusForbidden, nil } - file, err := files.NewFileInfo(files.FileOptions{ + file, err := files.FileInfoFaster(files.FileOptions{ Fs: d.user.Fs, Path: r.URL.Path, Modify: d.user.Perm.Modify, @@ -102,7 +96,7 @@ func resourcePostHandler(fileCache FileCache) handleFunc { return errToStatus(err), err } - file, err := files.NewFileInfo(files.FileOptions{ + file, err := files.FileInfoFaster(files.FileOptions{ Fs: d.user.Fs, Path: r.URL.Path, Modify: d.user.Perm.Modify, @@ -308,7 +302,7 @@ func patchAction(ctx context.Context, action, src, dst string, d *data, fileCach src = path.Clean("/" + src) dst = path.Clean("/" + dst) - file, err := files.NewFileInfo(files.FileOptions{ + file, err := files.FileInfoFaster(files.FileOptions{ Fs: d.user.Fs, Path: src, Modify: d.user.Perm.Modify, @@ -338,14 +332,13 @@ type DiskUsageResponse struct { } var diskUsage = withUser(func(w http.ResponseWriter, r *http.Request, d *data) (int, error) { - file, err := files.NewFileInfo(files.FileOptions{ + file, err := files.FileInfoFaster(files.FileOptions{ Fs: d.user.Fs, Path: r.URL.Path, Modify: d.user.Perm.Modify, Expand: false, ReadHeader: false, Checker: d, - Content: false, }) if err != nil { return errToStatus(err), err diff --git a/backend/http/search.go b/backend/http/search.go index 08908fbf..25aee389 100644 --- a/backend/http/search.go +++ b/backend/http/search.go @@ -4,7 +4,8 @@ import ( "net/http" "strings" - "github.com/gtsteffaniak/filebrowser/index" + "github.com/gtsteffaniak/filebrowser/files" + "github.com/gtsteffaniak/filebrowser/settings" ) var searchHandler = withUser(func(w http.ResponseWriter, r *http.Request, d *data) (int, error) { @@ -13,7 +14,7 @@ var searchHandler = withUser(func(w http.ResponseWriter, r *http.Request, d *dat // Retrieve the User-Agent and X-Auth headers from the request sessionId := r.Header.Get("SessionId") userScope := r.Header.Get("UserScope") - index := *index.GetIndex() + index := files.GetIndex(settings.Config.Server.Root) combinedScope := strings.TrimPrefix(userScope+r.URL.Path, ".") combinedScope = strings.TrimPrefix(combinedScope, "/") results, fileTypes := index.Search(query, combinedScope, sessionId) diff --git a/backend/http/settings.go b/backend/http/settings.go index 1acebf83..1185db8d 100644 --- a/backend/http/settings.go +++ b/backend/http/settings.go @@ -21,9 +21,9 @@ type settingsData struct { var settingsGetHandler = withAdmin(func(w http.ResponseWriter, r *http.Request, d *data) (int, error) { data := &settingsData{ - Signup: settings.GlobalConfiguration.Auth.Signup, - CreateUserDir: settings.GlobalConfiguration.Server.CreateUserDir, - UserHomeBasePath: settings.GlobalConfiguration.Server.UserHomeBasePath, + Signup: settings.Config.Auth.Signup, + CreateUserDir: settings.Config.Server.CreateUserDir, + UserHomeBasePath: settings.Config.Server.UserHomeBasePath, Defaults: d.settings.UserDefaults, Rules: d.settings.Rules, Frontend: d.settings.Frontend, diff --git a/backend/http/static.go b/backend/http/static.go index f2f7b860..a9ce5383 100644 --- a/backend/http/static.go +++ b/backend/http/static.go @@ -30,12 +30,12 @@ func handleWithStaticData(w http.ResponseWriter, _ *http.Request, d *data, fSys "Name": d.settings.Frontend.Name, "DisableExternal": d.settings.Frontend.DisableExternal, "DisableUsedPercentage": d.settings.Frontend.DisableUsedPercentage, - "darkMode": settings.GlobalConfiguration.UserDefaults.DarkMode, + "darkMode": settings.Config.UserDefaults.DarkMode, "Color": d.settings.Frontend.Color, "BaseURL": d.server.BaseURL, "Version": version.Version, "StaticURL": path.Join(d.server.BaseURL, "/static"), - "Signup": settings.GlobalConfiguration.Auth.Signup, + "Signup": settings.Config.Auth.Signup, "NoAuth": d.settings.Auth.Method == "noauth", "AuthMethod": d.settings.Auth.Method, "LoginPage": auther.LoginPage(), diff --git a/backend/http/users.go b/backend/http/users.go index 20765293..039e9efd 100644 --- a/backend/http/users.go +++ b/backend/http/users.go @@ -19,6 +19,11 @@ var ( NonModifiableFieldsForNonAdmin = []string{"Username", "Scope", "LockPassword", "Perm", "Commands", "Rules"} ) +// SortingSettings represents the sorting settings. +type Sorting struct { + By string `json:"by"` + Asc bool `json:"asc"` +} type modifyUserRequest struct { modifyRequest Data *users.User `json:"data"` diff --git a/backend/index/indexing.go b/backend/index/indexing.go deleted file mode 100644 index 0655fd36..00000000 --- a/backend/index/indexing.go +++ /dev/null @@ -1,138 +0,0 @@ -package index - -import ( - "log" - "os" - "slices" - "strings" - "sync" - "time" - - "github.com/gtsteffaniak/filebrowser/settings" -) - -type Index struct { - Dirs []string - Files []string -} - -var ( - rootPath string = "/srv" - indexes Index - indexMutex sync.RWMutex - lastIndexed time.Time -) - -func GetIndex() *Index { - return &indexes -} - -func Initialize(intervalMinutes uint32) { - // Initialize the index - indexes = Index{ - Dirs: []string{}, - Files: []string{}, - } - rootPath = settings.GlobalConfiguration.Server.Root - var numFiles, numDirs int - log.Println("Indexing files...") - lastIndexedStart := time.Now() - // Call the function to index files and directories - totalNumFiles, totalNumDirs, err := indexFiles(rootPath, &numFiles, &numDirs) - if err != nil { - log.Fatal(err) - } - lastIndexed = lastIndexedStart - go indexingScheduler(intervalMinutes) - log.Println("Successfully indexed files.") - log.Println("Files found :", totalNumFiles) - log.Println("Directories found :", totalNumDirs) -} - -func indexingScheduler(intervalMinutes uint32) { - log.Printf("Indexing scheduler will run every %v minutes", intervalMinutes) - for { - indexes.Dirs = slices.Compact(indexes.Dirs) - indexes.Files = slices.Compact(indexes.Files) - time.Sleep(time.Duration(intervalMinutes) * time.Minute) - var numFiles, numDirs int - lastIndexedStart := time.Now() - totalNumFiles, totalNumDirs, err := indexFiles(rootPath, &numFiles, &numDirs) - if err != nil { - log.Fatal(err) - } - lastIndexed = lastIndexedStart - if totalNumFiles+totalNumDirs > 0 { - log.Println("re-indexing found changes and updated the index.") - } - } -} - -func removeFromSlice(slice []string, target string) []string { - for i, s := range slice { - if s == target { - // Swap the target element with the last element - slice[i], slice[len(slice)-1] = slice[len(slice)-1], slice[i] - // Resize the slice to exclude the last element - slice = slice[:len(slice)-1] - break // Exit the loop, assuming there's only one target element - } - } - return slice -} - -// Define a function to recursively index files and directories -func indexFiles(path string, numFiles *int, numDirs *int) (int, int, error) { - // Check if the current directory has been modified since last indexing - dir, err := os.Open(path) - if err != nil { - // directory must have been deleted, remove from index - indexes.Dirs = removeFromSlice(indexes.Dirs, path) - indexes.Files = removeFromSlice(indexes.Files, path) - } - defer dir.Close() - dirInfo, err := dir.Stat() - if err != nil { - return *numFiles, *numDirs, err - } - // Compare the last modified time of the directory with the last indexed time - if dirInfo.ModTime().Before(lastIndexed) { - return *numFiles, *numDirs, nil - } - // Read the directory contents - files, err := dir.Readdir(-1) - if err != nil { - return *numFiles, *numDirs, err - } - // Iterate over the files and directories - for _, file := range files { - if file.IsDir() { - *numDirs++ - addToIndex(path, file.Name(), true) - _, _, err := indexFiles(path+"/"+file.Name(), numFiles, numDirs) // recursive - if err != nil { - log.Println("Could not index :", err) - } - } else { - *numFiles++ - addToIndex(path, file.Name(), false) - } - } - return *numFiles, *numDirs, nil -} - -func addToIndex(path string, fileName string, isDir bool) { - indexMutex.Lock() - defer indexMutex.Unlock() - path = strings.TrimPrefix(path, rootPath+"/") - path = strings.TrimSuffix(path, "/") - adjustedPath := path + "/" + fileName - if path == rootPath { - adjustedPath = fileName - } - if isDir { - indexes.Dirs = append(indexes.Dirs, adjustedPath) - } else { - indexes.Files = append(indexes.Files, adjustedPath) - } -} diff --git a/backend/settings/config.go b/backend/settings/config.go index 646b1ce8..bab4a9b2 100644 --- a/backend/settings/config.go +++ b/backend/settings/config.go @@ -3,21 +3,23 @@ package settings import ( "log" "os" + "strings" "github.com/goccy/go-yaml" "github.com/gtsteffaniak/filebrowser/users" ) -var GlobalConfiguration Settings +var Config Settings func Initialize(configFile string) { yamlData := loadConfigFile(configFile) - GlobalConfiguration = setDefaults() - err := yaml.Unmarshal(yamlData, &GlobalConfiguration) + Config = setDefaults() + err := yaml.Unmarshal(yamlData, &Config) if err != nil { log.Fatalf("Error unmarshaling YAML data: %v", err) } - GlobalConfiguration.UserDefaults.Perm = GlobalConfiguration.UserDefaults.Permissions + Config.UserDefaults.Perm = Config.UserDefaults.Permissions + Config.Server.Root = strings.TrimSuffix(Config.Server.Root, "/") } func loadConfigFile(configFile string) []byte { @@ -25,7 +27,7 @@ func loadConfigFile(configFile string) []byte { yamlFile, err := os.Open(configFile) if err != nil { log.Printf("ERROR: opening config file\n %v\n WARNING: Using default config only\n If this was a mistake, please make sure the file exists and is accessible by the filebrowser binary.\n\n", err) - GlobalConfiguration = setDefaults() + Config = setDefaults() return []byte{} } defer yamlFile.Close() diff --git a/backend/settings/settings_test.go b/backend/settings/settings_test.go index 2753b184..26f30b7c 100644 --- a/backend/settings/settings_test.go +++ b/backend/settings/settings_test.go @@ -12,14 +12,14 @@ func TestConfigLoadChanged(t *testing.T) { yamlData := loadConfigFile("./testingConfig.yaml") // Marshal the YAML data to a more human-readable format newConfig := setDefaults() - GlobalConfiguration := setDefaults() + Config := setDefaults() err := yaml.Unmarshal(yamlData, &newConfig) if err != nil { log.Fatalf("Error unmarshaling YAML data: %v", err) } // Use go-cmp to compare the two structs - if diff := cmp.Diff(newConfig, GlobalConfiguration); diff == "" { + if diff := cmp.Diff(newConfig, Config); diff == "" { t.Errorf("No change when there should have been (-want +got):\n%s", diff) } } @@ -28,7 +28,7 @@ func TestConfigLoadSpecificValues(t *testing.T) { yamlData := loadConfigFile("./testingConfig.yaml") // Marshal the YAML data to a more human-readable format newConfig := setDefaults() - GlobalConfiguration := setDefaults() + Config := setDefaults() err := yaml.Unmarshal(yamlData, &newConfig) if err != nil { @@ -39,16 +39,16 @@ func TestConfigLoadSpecificValues(t *testing.T) { globalVal interface{} newVal interface{} }{ - {"Auth.Method", GlobalConfiguration.Auth.Method, newConfig.Auth.Method}, - {"Auth.Method", GlobalConfiguration.Auth.Method, newConfig.Auth.Method}, - {"Frontend.disableExternal", GlobalConfiguration.Frontend.DisableExternal, newConfig.Frontend.DisableExternal}, - {"UserDefaults.HideDotfiles", GlobalConfiguration.UserDefaults.HideDotfiles, newConfig.UserDefaults.HideDotfiles}, - {"Server.Database", GlobalConfiguration.Server.Database, newConfig.Server.Database}, + {"Auth.Method", Config.Auth.Method, newConfig.Auth.Method}, + {"Auth.Method", Config.Auth.Method, newConfig.Auth.Method}, + {"Frontend.disableExternal", Config.Frontend.DisableExternal, newConfig.Frontend.DisableExternal}, + {"UserDefaults.HideDotfiles", Config.UserDefaults.HideDotfiles, newConfig.UserDefaults.HideDotfiles}, + {"Server.Database", Config.Server.Database, newConfig.Server.Database}, } for _, tc := range testCases { if tc.globalVal == tc.newVal { - t.Errorf("Differences should have been found:\n\tGlobalConfig.%s: %v \n\tSetConfig: %v \n", tc.fieldName, tc.globalVal, tc.newVal) + t.Errorf("Differences should have been found:\n\tConfig.%s: %v \n\tSetConfig: %v \n", tc.fieldName, tc.globalVal, tc.newVal) } } } diff --git a/backend/users/users.go b/backend/users/users.go index 50ef124b..bdd0d30c 100644 --- a/backend/users/users.go +++ b/backend/users/users.go @@ -6,7 +6,6 @@ import ( "github.com/spf13/afero" - "github.com/gtsteffaniak/filebrowser/files" "github.com/gtsteffaniak/filebrowser/rules" ) @@ -21,25 +20,31 @@ type Permissions struct { Download bool `json:"download"` } +// SortingSettings represents the sorting settings. +type Sorting struct { + By string `json:"by"` + Asc bool `json:"asc"` +} + // User describes a user. type User struct { - DarkMode bool `json:"darkMode"` - DisableSettings bool `json:"disableSettings"` - ID uint `storm:"id,increment" json:"id"` - Username string `storm:"unique" json:"username"` - Password string `json:"password"` - Scope string `json:"scope"` - Locale string `json:"locale"` - LockPassword bool `json:"lockPassword"` - ViewMode string `json:"viewMode"` - SingleClick bool `json:"singleClick"` - Perm Permissions `json:"perm"` - Commands []string `json:"commands"` - Sorting files.Sorting `json:"sorting"` - Fs afero.Fs `json:"-" yaml:"-"` - Rules []rules.Rule `json:"rules"` - HideDotfiles bool `json:"hideDotfiles"` - DateFormat bool `json:"dateFormat"` + DarkMode bool `json:"darkMode"` + DisableSettings bool `json:"disableSettings"` + ID uint `storm:"id,increment" json:"id"` + Username string `storm:"unique" json:"username"` + Password string `json:"password"` + Scope string `json:"scope"` + Locale string `json:"locale"` + LockPassword bool `json:"lockPassword"` + ViewMode string `json:"viewMode"` + SingleClick bool `json:"singleClick"` + Perm Permissions `json:"perm"` + Commands []string `json:"commands"` + Sorting Sorting `json:"sorting"` + Fs afero.Fs `json:"-" yaml:"-"` + Rules []rules.Rule `json:"rules"` + HideDotfiles bool `json:"hideDotfiles"` + DateFormat bool `json:"dateFormat"` } // GetRules implements rules.Provider. diff --git a/backend/version/version.go b/backend/version/version.go index fb0df987..86e67b04 100644 --- a/backend/version/version.go +++ b/backend/version/version.go @@ -2,7 +2,7 @@ package version var ( // Version is the current File Browser version. - Version = "(0.2.1)" + Version = "(0.2.2)" // CommitSHA is the commmit sha. CommitSHA = "(unknown)" ) diff --git a/frontend/src/components/files/ListingItem.vue b/frontend/src/components/files/ListingItem.vue index 1302e45a..d53716d9 100644 --- a/frontend/src/components/files/ListingItem.vue +++ b/frontend/src/components/files/ListingItem.vue @@ -1,5 +1,6 @@ + +