Skip to content

Add subdirectory option to virtual folders #2037

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tecosaur wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add subdirectory option to virtual folders #2037

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions examples/ldapauthserver/httpd/models.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,9 @@ type SFTPGoFilesystem struct {
}

type virtualFolder struct {
VirtualPath string `json:"virtual_path"`
MappedPath string `json:"mapped_path"`
VirtualPath string `json:"virtual_path"`
MappedSubdirectory string `json:"mapped_subdirectory,omitempty"`
MappedPath string `json:"mapped_path"`
}

// SFTPGoUser defines an SFTPGo user
Expand Down
12 changes: 12 additions & 0 deletions examples/php-activedirectory-http-server/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,18 @@ $virtual_folders['example'] = [
"quota_files" => -1
]
];

// Alternative example using mapped_subdirectory for multi-tenant shared storage:
$virtual_folders['shared_example'] = [
[
"name" => "shared-documents",
"mapped_path" => 'F:\files\shared\documents',
"virtual_path" => "/documents",
"mapped_subdirectory" => "#USERNAME#",
"quota_size" => -1,
"quota_files" => -1
]
];
```

## Example Connection "Output Object" Allowing For No Files in the User's Home Directory ("Root Directory") but Allowing for Files in the Public/Private Virtual Folders
Expand Down
16 changes: 9 additions & 7 deletions internal/dataprovider/bolt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import (
)

const (
boltDatabaseVersion = 34
boltDatabaseVersion = 35
)

var (
Expand Down Expand Up @@ -3170,11 +3170,10 @@ func (p *BoltProvider) migrateDatabase() error {
providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err)
return err
case version == 33:
logger.InfoToConsole("updating database schema version: %d -> 34", version)
providerLog(logger.LevelInfo, "updating database schema version: %d -> 34", version)
return updateBoltDatabaseVersion(p.dbHandle, 34)

case version == 33, version == 34:
logger.InfoToConsole("updating database schema version: %d -> 35", version)
providerLog(logger.LevelInfo, "updating database schema version: %d -> 35", version)
return updateBoltDatabaseVersion(p.dbHandle, 35)
default:
if version > boltDatabaseVersion {
providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version,
Expand All @@ -3200,7 +3199,10 @@ func (p *BoltProvider) revertDatabase(targetVersion int) error {
logger.InfoToConsole("downgrading database schema version: %d -> 33", dbVersion.Version)
providerLog(logger.LevelInfo, "downgrading database schema version: %d -> 33", dbVersion.Version)
return updateBoltDatabaseVersion(p.dbHandle, 33)

case 35:
logger.InfoToConsole("downgrading database schema version: %d -> 34", dbVersion.Version)
providerLog(logger.LevelInfo, "downgrading database schema version: %d -> 34", dbVersion.Version)
return updateBoltDatabaseVersion(p.dbHandle, 34)
default:
return fmt.Errorf("database schema version not handled: %v", dbVersion.Version)
}
Expand Down
71 changes: 63 additions & 8 deletions internal/dataprovider/dataprovider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,10 @@ var (
// ErrShareUsageExceeded is returned when reserving share usage tokens would exceed the share max_tokens limit
ErrShareUsageExceeded = util.NewI18nError(
util.NewRecordNotFoundError("max share usage exceeded"), util.I18nErrorShareUsage)
errInvalidInput = util.NewValidationError("Invalid input. Slashes (/ ), colons (:), control characters, and reserved system names are not allowed")
errInvalidInput = util.NewValidationError("Invalid input. Slashes (/ ), colons (:), control characters, and reserved system names are not allowed")
errInvalidMappedSubdirectory = util.NewI18nError(
util.NewValidationError("mapped subdirectory must be a relative path that does not escape the mapped root"),
util.I18nErrorPathInvalid)
tz = ""
isAdminCreated atomic.Bool
validTLSUsernames = []string{string(sdk.TLSUsernameNone), string(sdk.TLSUsernameCN)}
Expand Down Expand Up @@ -2772,7 +2775,7 @@ func validateAssociatedVirtualFolders(vfolders []vfs.VirtualFolder) ([]vfs.Virtu
return []vfs.VirtualFolder{}, nil
}
var virtualFolders []vfs.VirtualFolder
folderNames := make(map[string]bool)
folderKeys := make(map[[2]string]bool)

for _, v := range vfolders {
v.Name = config.convertName(v.Name)
Expand All @@ -2789,9 +2792,18 @@ func validateAssociatedVirtualFolders(vfolders []vfs.VirtualFolder) ([]vfs.Virtu
if v.Name == "" {
return nil, util.NewI18nError(util.NewValidationError("folder name is mandatory"), util.I18nErrorFolderNameRequired)
}
if folderNames[v.Name] {
subdir, err := normalizeMappedSubdirectory(v.MappedSubdirectory)
if err != nil {
return nil, err
}
v.MappedSubdirectory = subdir
if err := validateMappedSubdirectoryQuota(v); err != nil {
return nil, err
}
folderKey := [2]string{v.Name, subdir}
if folderKeys[folderKey] {
return nil, util.NewI18nError(
util.NewValidationError(fmt.Sprintf("the folder %q is duplicated", v.Name)),
util.NewValidationError(duplicatedFolderMessage(v.Name, subdir)),
util.I18nErrorDuplicatedFolders,
)
}
Expand All @@ -2808,15 +2820,58 @@ func validateAssociatedVirtualFolders(vfolders []vfs.VirtualFolder) ([]vfs.Virtu
BaseVirtualFolder: vfs.BaseVirtualFolder{
Name: v.Name,
},
VirtualPath: cleanedVPath,
QuotaSize: v.QuotaSize,
QuotaFiles: v.QuotaFiles,
VirtualPath: cleanedVPath,
MappedSubdirectory: subdir,
QuotaSize: v.QuotaSize,
QuotaFiles: v.QuotaFiles,
})
folderNames[v.Name] = true
folderKeys[folderKey] = true
}
return virtualFolders, nil
}

func duplicatedFolderMessage(name, subdir string) string {
if subdir != "" {
return fmt.Sprintf("the folder %q with subdirectory %q is duplicated", name, subdir)
}
return fmt.Sprintf("the folder %q is duplicated", name)
}

func normalizeMappedSubdirectory(value string) (string, error) {
if value == "" {
return "", nil
}
value = strings.ReplaceAll(value, "\\", "/") // collapse "\" like CleanPath, so it can't evade the checks below
if path.IsAbs(value) {
return "", errInvalidMappedSubdirectory
}
clean := path.Clean(value)
if clean == "." {
return "", nil
}
if clean == ".." || strings.HasPrefix(clean, "../") {
return "", errInvalidMappedSubdirectory
}
// capped uniformly so a config stays valid across providers: the column is
// part of a unique index and MySQL stores it as varchar(255)
if len(clean) > 255 {
return "", util.NewI18nError(
util.NewValidationError("mapped subdirectory is too long, 255 is the maximum length allowed"),
util.I18nErrorPathInvalid)
}
return clean, nil
}

func validateMappedSubdirectoryQuota(v vfs.VirtualFolder) error {
if v.MappedSubdirectory != "" && (v.QuotaSize > 0 || v.QuotaFiles > 0) {
return util.NewI18nError(
util.NewValidationError("a positive quota applies to the whole folder, not the mapped subdirectory: use -1 to include it in the user quota, or 0 for no limit"),
util.I18nErrorFolderQuotaInvalid,
)
}
return nil
}

func validateUserTOTPConfig(c *UserTOTPConfig, username string) error {
if !c.Enabled {
c.ConfigName = ""
Expand Down
52 changes: 51 additions & 1 deletion internal/dataprovider/mysql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,18 @@ const (
"CREATE INDEX `{{prefix}}shares_groups_mapping_share_id_idx` ON `{{shares_groups_mapping}}` (`share_id`); " +
"CREATE INDEX `{{prefix}}shares_groups_mapping_group_id_idx` ON `{{shares_groups_mapping}}` (`group_id`);"
mysqlV34DownSQL = "DROP TABLE IF EXISTS `{{shares_groups_mapping}}`;"
mysqlV35SQL = "ALTER TABLE `{{users_folders_mapping}}` ADD COLUMN `subdirectory` varchar(255) NOT NULL DEFAULT '';" +
"ALTER TABLE `{{groups_folders_mapping}}` ADD COLUMN `subdirectory` varchar(255) NOT NULL DEFAULT '';" +
"ALTER TABLE `{{users_folders_mapping}}` DROP CONSTRAINT `{{prefix}}unique_user_folder_mapping`;" +
"ALTER TABLE `{{users_folders_mapping}}` ADD CONSTRAINT `{{prefix}}unique_user_folder_mapping` UNIQUE (`user_id`, `folder_id`, `subdirectory`);" +
"ALTER TABLE `{{groups_folders_mapping}}` DROP CONSTRAINT `{{prefix}}unique_group_folder_mapping`;" +
"ALTER TABLE `{{groups_folders_mapping}}` ADD CONSTRAINT `{{prefix}}unique_group_folder_mapping` UNIQUE (`group_id`, `folder_id`, `subdirectory`);"
mysqlV35DownSQL = "ALTER TABLE `{{users_folders_mapping}}` DROP CONSTRAINT `{{prefix}}unique_user_folder_mapping`;" +
"ALTER TABLE `{{groups_folders_mapping}}` DROP CONSTRAINT `{{prefix}}unique_group_folder_mapping`;" +
"ALTER TABLE `{{users_folders_mapping}}` ADD CONSTRAINT `{{prefix}}unique_user_folder_mapping` UNIQUE (`user_id`, `folder_id`);" +
"ALTER TABLE `{{groups_folders_mapping}}` ADD CONSTRAINT `{{prefix}}unique_group_folder_mapping` UNIQUE (`group_id`, `folder_id`);" +
"ALTER TABLE `{{users_folders_mapping}}` DROP COLUMN `subdirectory`;" +
"ALTER TABLE `{{groups_folders_mapping}}` DROP COLUMN `subdirectory`;"
)

// MySQLProvider defines the auth provider for MySQL/MariaDB database
Expand Down Expand Up @@ -817,6 +829,8 @@ func (p *MySQLProvider) migrateDatabase() error {
return err
case version == 33:
return updateMySQLDatabaseFromV33(p.dbHandle)
case version == 34:
return updateMySQLDatabaseFromV34(p.dbHandle)
default:
if version > sqlDatabaseVersion {
providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version,
Expand All @@ -841,6 +855,8 @@ func (p *MySQLProvider) revertDatabase(targetVersion int) error {
switch dbVersion.Version {
case 34:
return downgradeMySQLDatabaseFromV34(p.dbHandle)
case 35:
return downgradeMySQLDatabaseFromV35(p.dbHandle)
default:
return fmt.Errorf("database schema version not handled: %d", dbVersion.Version)
}
Expand Down Expand Up @@ -880,13 +896,27 @@ func (p *MySQLProvider) normalizeError(err error, fieldType int) error {
}

func updateMySQLDatabaseFromV33(dbHandle *sql.DB) error {
return updateMySQLDatabaseFrom33To34(dbHandle)
if err := updateMySQLDatabaseFrom33To34(dbHandle); err != nil {
return err
}
return updateMySQLDatabaseFromV34(dbHandle)
}

func updateMySQLDatabaseFromV34(dbHandle *sql.DB) error {
return updateMySQLDatabaseFrom34To35(dbHandle)
}

func downgradeMySQLDatabaseFromV34(dbHandle *sql.DB) error {
return downgradeMySQLDatabaseFrom34To33(dbHandle)
}

func downgradeMySQLDatabaseFromV35(dbHandle *sql.DB) error {
if err := downgradeMySQLDatabaseFrom35To34(dbHandle); err != nil {
return err
}
return downgradeMySQLDatabaseFromV34(dbHandle)
}

func updateMySQLDatabaseFrom33To34(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 33 -> 34")
providerLog(logger.LevelInfo, "updating database schema version: 33 -> 34")
Expand All @@ -905,3 +935,23 @@ func downgradeMySQLDatabaseFrom34To33(dbHandle *sql.DB) error {
sql := strings.ReplaceAll(mysqlV34DownSQL, "{{shares_groups_mapping}}", sqlTableSharesGroupsMapping)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 33, false)
}

func updateMySQLDatabaseFrom34To35(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 34 -> 35")
providerLog(logger.LevelInfo, "updating database schema version: 34 -> 35")

sql := strings.ReplaceAll(mysqlV35SQL, "{{prefix}}", config.SQLTablesPrefix)
sql = strings.ReplaceAll(sql, "{{users_folders_mapping}}", sqlTableUsersFoldersMapping)
sql = strings.ReplaceAll(sql, "{{groups_folders_mapping}}", sqlTableGroupsFoldersMapping)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 35, true)
}

func downgradeMySQLDatabaseFrom35To34(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 35 -> 34")
providerLog(logger.LevelInfo, "downgrading database schema version: 35 -> 34")

sql := strings.ReplaceAll(mysqlV35DownSQL, "{{prefix}}", config.SQLTablesPrefix)
sql = strings.ReplaceAll(sql, "{{users_folders_mapping}}", sqlTableUsersFoldersMapping)
sql = strings.ReplaceAll(sql, "{{groups_folders_mapping}}", sqlTableGroupsFoldersMapping)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 34, false)
}
52 changes: 51 additions & 1 deletion internal/dataprovider/pgsql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,18 @@ CREATE INDEX "{{prefix}}shares_groups_mapping_share_id_idx" ON "{{shares_groups_
CREATE INDEX "{{prefix}}shares_groups_mapping_group_id_idx" ON "{{shares_groups_mapping}}" ("group_id");
`
pgsqlV34DownSQL = `DROP TABLE IF EXISTS "{{shares_groups_mapping}}";`
pgsqlV35SQL = `ALTER TABLE "{{users_folders_mapping}}" ADD COLUMN "subdirectory" text NOT NULL DEFAULT '';
ALTER TABLE "{{groups_folders_mapping}}" ADD COLUMN "subdirectory" text NOT NULL DEFAULT '';
ALTER TABLE "{{users_folders_mapping}}" DROP CONSTRAINT "{{prefix}}unique_user_folder_mapping";
ALTER TABLE "{{users_folders_mapping}}" ADD CONSTRAINT "{{prefix}}unique_user_folder_mapping" UNIQUE ("user_id", "folder_id", "subdirectory");
ALTER TABLE "{{groups_folders_mapping}}" DROP CONSTRAINT "{{prefix}}unique_group_folder_mapping";
ALTER TABLE "{{groups_folders_mapping}}" ADD CONSTRAINT "{{prefix}}unique_group_folder_mapping" UNIQUE ("group_id", "folder_id", "subdirectory");`
pgsqlV35DownSQL = `ALTER TABLE "{{users_folders_mapping}}" DROP CONSTRAINT "{{prefix}}unique_user_folder_mapping";
ALTER TABLE "{{groups_folders_mapping}}" DROP CONSTRAINT "{{prefix}}unique_group_folder_mapping";
ALTER TABLE "{{users_folders_mapping}}" ADD CONSTRAINT "{{prefix}}unique_user_folder_mapping" UNIQUE ("user_id", "folder_id");
ALTER TABLE "{{groups_folders_mapping}}" ADD CONSTRAINT "{{prefix}}unique_group_folder_mapping" UNIQUE ("group_id", "folder_id");
ALTER TABLE "{{users_folders_mapping}}" DROP COLUMN "subdirectory" CASCADE;
ALTER TABLE "{{groups_folders_mapping}}" DROP COLUMN "subdirectory" CASCADE;`
)

var (
Expand Down Expand Up @@ -845,6 +857,8 @@ func (p *PGSQLProvider) migrateDatabase() error {
return err
case version == 33:
return updatePGSQLDatabaseFromV33(p.dbHandle)
case version == 34:
return updatePGSQLDatabaseFromV34(p.dbHandle)
default:
if version > sqlDatabaseVersion {
providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version,
Expand All @@ -869,6 +883,8 @@ func (p *PGSQLProvider) revertDatabase(targetVersion int) error {
switch dbVersion.Version {
case 34:
return downgradePGSQLDatabaseFromV34(p.dbHandle)
case 35:
return downgradePGSQLDatabaseFromV35(p.dbHandle)
default:
return fmt.Errorf("database schema version not handled: %d", dbVersion.Version)
}
Expand Down Expand Up @@ -908,13 +924,27 @@ func (p *PGSQLProvider) normalizeError(err error, fieldType int) error {
}

func updatePGSQLDatabaseFromV33(dbHandle *sql.DB) error {
return updatePGSQLDatabaseFrom33To34(dbHandle)
if err := updatePGSQLDatabaseFrom33To34(dbHandle); err != nil {
return err
}
return updatePGSQLDatabaseFromV34(dbHandle)
}

func updatePGSQLDatabaseFromV34(dbHandle *sql.DB) error {
return updatePGSQLDatabaseFrom34To35(dbHandle)
}

func downgradePGSQLDatabaseFromV34(dbHandle *sql.DB) error {
return downgradePGSQLDatabaseFrom34To33(dbHandle)
}

func downgradePGSQLDatabaseFromV35(dbHandle *sql.DB) error {
if err := downgradePGSQLDatabaseFrom35To34(dbHandle); err != nil {
return err
}
return downgradePGSQLDatabaseFromV34(dbHandle)
}

func updatePGSQLDatabaseFrom33To34(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 33 -> 34")
providerLog(logger.LevelInfo, "updating database schema version: 33 -> 34")
Expand All @@ -933,3 +963,23 @@ func downgradePGSQLDatabaseFrom34To33(dbHandle *sql.DB) error {
sql := strings.ReplaceAll(pgsqlV34DownSQL, "{{shares_groups_mapping}}", sqlTableSharesGroupsMapping)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 33, false)
}

func updatePGSQLDatabaseFrom34To35(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 34 -> 35")
providerLog(logger.LevelInfo, "updating database schema version: 34 -> 35")

sql := strings.ReplaceAll(pgsqlV35SQL, "{{prefix}}", config.SQLTablesPrefix)
sql = strings.ReplaceAll(sql, "{{users_folders_mapping}}", sqlTableUsersFoldersMapping)
sql = strings.ReplaceAll(sql, "{{groups_folders_mapping}}", sqlTableGroupsFoldersMapping)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 35, true)
}

func downgradePGSQLDatabaseFrom35To34(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 35 -> 34")
providerLog(logger.LevelInfo, "downgrading database schema version: 35 -> 34")

sql := strings.ReplaceAll(pgsqlV35DownSQL, "{{prefix}}", config.SQLTablesPrefix)
sql = strings.ReplaceAll(sql, "{{users_folders_mapping}}", sqlTableUsersFoldersMapping)
sql = strings.ReplaceAll(sql, "{{groups_folders_mapping}}", sqlTableGroupsFoldersMapping)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 34, false)
}
10 changes: 5 additions & 5 deletions internal/dataprovider/sqlcommon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import (
)

const (
sqlDatabaseVersion = 34
sqlDatabaseVersion = 35
defaultSQLQueryTimeout = 10 * time.Second
longSQLQueryTimeout = 60 * time.Second
)
Expand Down Expand Up @@ -2543,7 +2543,7 @@ func sqlCommonClearUserGroupMapping(ctx context.Context, user *User, dbHandle sq

func sqlCommonAddUserFolderMapping(ctx context.Context, user *User, folder *vfs.VirtualFolder, sortOrder int, dbHandle sqlQuerier) error {
q := getAddUserFolderMappingQuery()
_, err := dbHandle.ExecContext(ctx, q, folder.VirtualPath, folder.QuotaSize, folder.QuotaFiles, folder.Name, user.Username, sortOrder)
_, err := dbHandle.ExecContext(ctx, q, folder.VirtualPath, folder.MappedSubdirectory, folder.QuotaSize, folder.QuotaFiles, folder.Name, user.Username, sortOrder)
return err
}

Expand All @@ -2557,7 +2557,7 @@ func sqlCommonAddGroupFolderMapping(ctx context.Context, group *Group, folder *v
dbHandle sqlQuerier,
) error {
q := getAddGroupFolderMappingQuery()
_, err := dbHandle.ExecContext(ctx, q, folder.VirtualPath, folder.QuotaSize, folder.QuotaFiles, folder.Name, group.Name, sortOrder)
_, err := dbHandle.ExecContext(ctx, q, folder.VirtualPath, folder.MappedSubdirectory, folder.QuotaSize, folder.QuotaFiles, folder.Name, group.Name, sortOrder)
return err
}

Expand Down Expand Up @@ -2765,7 +2765,7 @@ func getUsersWithVirtualFolders(ctx context.Context, users []User, dbHandle sqlQ
var mappedPath, description sql.NullString
var fsConfig []byte
err = rows.Scan(&folder.ID, &folder.Name, &mappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles,
&folder.LastQuotaUpdate, &folder.VirtualPath, &folder.QuotaSize, &folder.QuotaFiles, &userID, &fsConfig,
&folder.LastQuotaUpdate, &folder.VirtualPath, &folder.MappedSubdirectory, &folder.QuotaSize, &folder.QuotaFiles, &userID, &fsConfig,
&description)
if err != nil {
return users, err
Expand Down Expand Up @@ -2916,7 +2916,7 @@ func getGroupsWithVirtualFolders(ctx context.Context, groups []Group, dbHandle s
var mappedPath, description sql.NullString
var fsConfig []byte
err = rows.Scan(&folder.ID, &folder.Name, &mappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles,
&folder.LastQuotaUpdate, &folder.VirtualPath, &folder.QuotaSize, &folder.QuotaFiles, &groupID, &fsConfig,
&folder.LastQuotaUpdate, &folder.VirtualPath, &folder.MappedSubdirectory, &folder.QuotaSize, &folder.QuotaFiles, &groupID, &fsConfig,
&description)
if err != nil {
return groups, err
Expand Down
Loading