1
0
mirror of https://github.com/lus/pasty.git synced 2023-08-10 21:13:09 +03:00

restructure startup & config logic

This commit is contained in:
Lukas Schulte Pelkum
2023-06-07 17:46:19 +02:00
parent e93b292daf
commit 3575c02c1e
21 changed files with 175 additions and 1412 deletions

View File

@ -1,24 +1,18 @@
package storage
import (
"fmt"
"context"
"github.com/lus/pasty/internal/config"
"strings"
"github.com/lus/pasty/internal/config"
"github.com/lus/pasty/internal/paste"
"github.com/lus/pasty/internal/storage/file"
"github.com/lus/pasty/internal/storage/mongodb"
"github.com/lus/pasty/internal/storage/postgres"
"github.com/lus/pasty/internal/storage/s3"
)
// Current holds the current storage driver
var Current Driver
// Driver represents a storage driver
type Driver interface {
Initialize() error
Terminate() error
Initialize(ctx context.Context, cfg *config.Config) error
Close() error
ListIDs() ([]string, error)
Get(id string) (*paste.Paste, error)
Save(paste *paste.Paste) error
@ -26,35 +20,12 @@ type Driver interface {
Cleanup() (int, error)
}
// Load loads the current storage driver
func Load() error {
// Define the driver to use
driver, err := GetDriver(config.Current.StorageType)
if err != nil {
return err
}
// Initialize the driver
err = driver.Initialize()
if err != nil {
return err
}
Current = driver
return nil
}
// GetDriver returns the driver with the given type if it exists
func GetDriver(storageType string) (Driver, error) {
switch strings.TrimSpace(strings.ToLower(storageType)) {
case "file":
return new(file.FileDriver), nil
// ResolveDriver returns the driver with the given name if it exists
func ResolveDriver(name string) (Driver, bool) {
switch strings.TrimSpace(strings.ToLower(name)) {
case "postgres":
return new(postgres.PostgresDriver), nil
case "mongodb":
return new(mongodb.MongoDBDriver), nil
case "s3":
return new(s3.S3Driver), nil
return new(postgres.Driver), true
default:
return nil, fmt.Errorf("invalid storage type '%s'", storageType)
return nil, false
}
}

View File

@ -1,145 +0,0 @@
package file
import (
"encoding/base64"
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
"github.com/lus/pasty/internal/config"
"github.com/lus/pasty/internal/paste"
)
// FileDriver represents the file storage driver
type FileDriver struct {
filePath string
}
// Initialize initializes the file storage driver
func (driver *FileDriver) Initialize() error {
driver.filePath = config.Current.File.Path
return os.MkdirAll(driver.filePath, os.ModePerm)
}
// Terminate terminates the file storage driver (does nothing, because the file storage driver does not need any termination)
func (driver *FileDriver) Terminate() error {
return nil
}
// ListIDs returns a list of all existing paste IDs
func (driver *FileDriver) ListIDs() ([]string, error) {
// Define the IDs slice
var ids []string
// Fill the IDs slice
err := filepath.Walk(driver.filePath, func(_ string, info os.FileInfo, err error) error {
// Check if a walking error occurred
if err != nil {
return err
}
// Only count JSON files
if !strings.HasSuffix(info.Name(), ".json") {
return nil
}
// Decode the file name
decoded, err := base64.StdEncoding.DecodeString(strings.TrimSuffix(info.Name(), ".json"))
if err != nil {
return err
}
// Append the ID to the IDs slice
ids = append(ids, string(decoded))
return nil
})
if err != nil {
return nil, err
}
// Return the IDs slice
return ids, nil
}
// Get loads a paste
func (driver *FileDriver) Get(id string) (*paste.Paste, error) {
// Read the file
id = base64.StdEncoding.EncodeToString([]byte(id))
data, err := ioutil.ReadFile(filepath.Join(driver.filePath, id+".json"))
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
// Unmarshal the file into a paste
paste := new(paste.Paste)
err = json.Unmarshal(data, &paste)
if err != nil {
return nil, err
}
return paste, nil
}
// Save saves a paste
func (driver *FileDriver) Save(paste *paste.Paste) error {
// Marshal the paste
jsonBytes, err := json.Marshal(paste)
if err != nil {
return err
}
// Create the file to save the paste to
id := base64.StdEncoding.EncodeToString([]byte(paste.ID))
file, err := os.Create(filepath.Join(driver.filePath, id+".json"))
if err != nil {
return err
}
defer file.Close()
// Write the JSON data into the file
_, err = file.Write(jsonBytes)
return err
}
// Delete deletes a paste
func (driver *FileDriver) Delete(id string) error {
id = base64.StdEncoding.EncodeToString([]byte(id))
return os.Remove(filepath.Join(driver.filePath, id+".json"))
}
// Cleanup cleans up the expired pastes
func (driver *FileDriver) Cleanup() (int, error) {
// Retrieve all paste IDs
ids, err := driver.ListIDs()
if err != nil {
return 0, err
}
// Define the amount of deleted items
deleted := 0
// Loop through all pastes
for _, id := range ids {
// Retrieve the paste object
paste, err := driver.Get(id)
if err != nil {
return deleted, err
}
// Delete the paste if it is expired
lifetime := config.Current.AutoDelete.Lifetime
if paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() {
err = driver.Delete(id)
if err != nil {
return deleted, err
}
deleted++
}
}
return deleted, nil
}

View File

@ -1,20 +0,0 @@
package storage
import (
"github.com/lus/pasty/internal/config"
"github.com/lus/pasty/internal/utils"
)
// AcquireID generates a new unique ID
func AcquireID() (string, error) {
for {
id := utils.RandomString(config.Current.IDCharacters, config.Current.IDLength)
paste, err := Current.Get(id)
if err != nil {
return "", err
}
if paste == nil {
return id, nil
}
}
}

View File

@ -1,171 +0,0 @@
package mongodb
import (
"context"
"time"
"github.com/lus/pasty/internal/config"
"github.com/lus/pasty/internal/paste"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
)
// MongoDBDriver represents the MongoDB storage driver
type MongoDBDriver struct {
client *mongo.Client
database string
collection string
}
// Initialize initializes the MongoDB storage driver
func (driver *MongoDBDriver) Initialize() error {
// Define the context for the following database operation
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
// Connect to the MongoDB host
client, err := mongo.Connect(ctx, options.Client().ApplyURI(config.Current.MongoDB.DSN))
if err != nil {
return err
}
// Ping the MongoDB host
err = client.Ping(ctx, readpref.Primary())
if err != nil {
return err
}
// Set the driver attributes
driver.client = client
driver.database = config.Current.MongoDB.Database
driver.collection = config.Current.MongoDB.Collection
return nil
}
// Terminate terminates the MongoDB storage driver
func (driver *MongoDBDriver) Terminate() error {
return driver.client.Disconnect(context.TODO())
}
// ListIDs returns a list of all existing paste IDs
func (driver *MongoDBDriver) ListIDs() ([]string, error) {
// Define the collection to use for this database operation
collection := driver.client.Database(driver.database).Collection(driver.collection)
// Define the context for the following database operation
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// Retrieve all paste documents
result, err := collection.Find(ctx, bson.M{})
if err != nil {
return nil, err
}
// Decode all paste documents
var pasteSlice []paste.Paste
err = result.All(ctx, &pasteSlice)
if err != nil {
return nil, err
}
// Read and return the IDs of all paste objects
var ids []string
for _, paste := range pasteSlice {
ids = append(ids, paste.ID)
}
return ids, nil
}
// Get loads a paste
func (driver *MongoDBDriver) Get(id string) (*paste.Paste, error) {
// Define the collection to use for this database operation
collection := driver.client.Database(driver.database).Collection(driver.collection)
// Define the context for the following database operation
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// Try to retrieve the corresponding paste document
filter := bson.M{"_id": id}
result := collection.FindOne(ctx, filter)
err := result.Err()
if err != nil {
if err == mongo.ErrNoDocuments {
return nil, nil
}
return nil, err
}
// Return the retrieved paste object
paste := new(paste.Paste)
err = result.Decode(paste)
if err != nil {
return nil, err
}
return paste, nil
}
// Save saves a paste
func (driver *MongoDBDriver) Save(paste *paste.Paste) error {
// Define the collection to use for this database operation
collection := driver.client.Database(driver.database).Collection(driver.collection)
// Define the context for the following database operation
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// Upsert the paste object
filter := bson.M{"_id": paste.ID}
_, err := collection.UpdateOne(ctx, filter, bson.M{"$set": paste}, options.Update().SetUpsert(true))
return err
}
// Delete deletes a paste
func (driver *MongoDBDriver) Delete(id string) error {
// Define the collection to use for this database operation
collection := driver.client.Database(driver.database).Collection(driver.collection)
// Define the context for the following database operation
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// Delete the document
filter := bson.M{"_id": id}
_, err := collection.DeleteOne(ctx, filter)
return err
}
// Cleanup cleans up the expired pastes
func (driver *MongoDBDriver) Cleanup() (int, error) {
// Retrieve all paste IDs
ids, err := driver.ListIDs()
if err != nil {
return 0, err
}
// Define the amount of deleted items
deleted := 0
// Loop through all pastes
for _, id := range ids {
// Retrieve the paste object
paste, err := driver.Get(id)
if err != nil {
return 0, err
}
// Delete the paste if it is expired
lifetime := config.Current.AutoDelete.Lifetime
if paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() {
err = driver.Delete(id)
if err != nil {
return 0, err
}
deleted++
}
}
return deleted, nil
}

View File

@ -18,14 +18,15 @@ import (
//go:embed migrations/*.sql
var migrations embed.FS
// PostgresDriver represents the Postgres storage driver
type PostgresDriver struct {
pool *pgxpool.Pool
// Driver represents the Postgres storage driver
type Driver struct {
pool *pgxpool.Pool
autoDeleteLifetime time.Duration
}
// Initialize initializes the Postgres storage driver
func (driver *PostgresDriver) Initialize() error {
pool, err := pgxpool.Connect(context.Background(), config.Current.Postgres.DSN)
func (driver *Driver) Initialize(ctx context.Context, cfg *config.Config) error {
pool, err := pgxpool.Connect(ctx, cfg.Postgres.DSN)
if err != nil {
return err
}
@ -35,7 +36,7 @@ func (driver *PostgresDriver) Initialize() error {
return err
}
migrator, err := migrate.NewWithSourceInstance("iofs", source, config.Current.Postgres.DSN)
migrator, err := migrate.NewWithSourceInstance("iofs", source, cfg.Postgres.DSN)
if err != nil {
return err
}
@ -45,17 +46,18 @@ func (driver *PostgresDriver) Initialize() error {
}
driver.pool = pool
driver.autoDeleteLifetime = cfg.AutoDelete.Lifetime
return nil
}
// Terminate terminates the Postgres storage driver
func (driver *PostgresDriver) Terminate() error {
// Close terminates the Postgres storage driver
func (driver *Driver) Close() error {
driver.pool.Close()
return nil
}
// ListIDs returns a list of all existing paste IDs
func (driver *PostgresDriver) ListIDs() ([]string, error) {
func (driver *Driver) ListIDs() ([]string, error) {
query := "SELECT id FROM pastes"
rows, err := driver.pool.Query(context.Background(), query)
@ -76,7 +78,7 @@ func (driver *PostgresDriver) ListIDs() ([]string, error) {
}
// Get loads a paste
func (driver *PostgresDriver) Get(id string) (*paste.Paste, error) {
func (driver *Driver) Get(id string) (*paste.Paste, error) {
query := "SELECT * FROM pastes WHERE id = $1"
row := driver.pool.QueryRow(context.Background(), query, id)
@ -92,7 +94,7 @@ func (driver *PostgresDriver) Get(id string) (*paste.Paste, error) {
}
// Save saves a paste
func (driver *PostgresDriver) Save(paste *paste.Paste) error {
func (driver *Driver) Save(paste *paste.Paste) error {
query := `
INSERT INTO pastes (id, content, "modificationToken", created, metadata)
VALUES ($1, $2, $3, $4, $5)
@ -108,7 +110,7 @@ func (driver *PostgresDriver) Save(paste *paste.Paste) error {
}
// Delete deletes a paste
func (driver *PostgresDriver) Delete(id string) error {
func (driver *Driver) Delete(id string) error {
query := "DELETE FROM pastes WHERE id = $1"
_, err := driver.pool.Exec(context.Background(), query, id)
@ -116,10 +118,10 @@ func (driver *PostgresDriver) Delete(id string) error {
}
// Cleanup cleans up the expired pastes
func (driver *PostgresDriver) Cleanup() (int, error) {
func (driver *Driver) Cleanup() (int, error) {
query := "DELETE FROM pastes WHERE created < $1"
tag, err := driver.pool.Exec(context.Background(), query, time.Now().Add(-config.Current.AutoDelete.Lifetime).Unix())
tag, err := driver.pool.Exec(context.Background(), query, time.Now().Add(-driver.autoDeleteLifetime).Unix())
if err != nil {
return 0, err
}

View File

@ -1,136 +0,0 @@
package s3
import (
"bytes"
"context"
"encoding/json"
"io/ioutil"
"strings"
"time"
"github.com/lus/pasty/internal/config"
"github.com/lus/pasty/internal/paste"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
// S3Driver represents the AWS S3 storage driver
type S3Driver struct {
client *minio.Client
bucket string
}
// Initialize initializes the AWS S3 storage driver
func (driver *S3Driver) Initialize() error {
client, err := minio.New(config.Current.S3.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(config.Current.S3.AccessKeyID, config.Current.S3.SecretAccessKey, config.Current.S3.SecretToken),
Secure: config.Current.S3.Secure,
Region: config.Current.S3.Region,
})
if err != nil {
return err
}
driver.client = client
driver.bucket = config.Current.S3.Bucket
return nil
}
// Terminate terminates the AWS S3 storage driver (does nothing, because the AWS S3 storage driver does not need any termination)
func (driver *S3Driver) Terminate() error {
return nil
}
// ListIDs returns a list of all existing paste IDs
func (driver *S3Driver) ListIDs() ([]string, error) {
// Define the IDs slice
var ids []string
// Fill the IDs slice
channel := driver.client.ListObjects(context.Background(), driver.bucket, minio.ListObjectsOptions{})
for object := range channel {
if object.Err != nil {
return nil, object.Err
}
ids = append(ids, strings.TrimSuffix(object.Key, ".json"))
}
// Return the IDs slice
return ids, nil
}
// Get loads a paste
func (driver *S3Driver) Get(id string) (*paste.Paste, error) {
// Read the object
object, err := driver.client.GetObject(context.Background(), driver.bucket, id+".json", minio.GetObjectOptions{})
if err != nil {
return nil, err
}
data, err := ioutil.ReadAll(object)
if err != nil {
if minio.ToErrorResponse(err).Code == "NoSuchKey" {
return nil, nil
}
return nil, err
}
// Unmarshal the object into a paste
paste := new(paste.Paste)
err = json.Unmarshal(data, &paste)
if err != nil {
return nil, err
}
return paste, nil
}
// Save saves a paste
func (driver *S3Driver) Save(paste *paste.Paste) error {
// Marshal the paste
jsonBytes, err := json.Marshal(paste)
if err != nil {
return err
}
// Put the object
reader := bytes.NewReader(jsonBytes)
_, err = driver.client.PutObject(context.Background(), driver.bucket, paste.ID+".json", reader, reader.Size(), minio.PutObjectOptions{
ContentType: "application/json",
})
return err
}
// Delete deletes a paste
func (driver *S3Driver) Delete(id string) error {
return driver.client.RemoveObject(context.Background(), driver.bucket, id+".json", minio.RemoveObjectOptions{})
}
// Cleanup cleans up the expired pastes
func (driver *S3Driver) Cleanup() (int, error) {
// Retrieve all paste IDs
ids, err := driver.ListIDs()
if err != nil {
return 0, err
}
// Define the amount of deleted items
deleted := 0
// Loop through all pastes
for _, id := range ids {
// Retrieve the paste object
paste, err := driver.Get(id)
if err != nil {
return 0, err
}
// Delete the paste if it is expired
lifetime := config.Current.AutoDelete.Lifetime
if paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() {
err = driver.Delete(id)
if err != nil {
return 0, err
}
deleted++
}
}
return deleted, nil
}