1
0
mirror of https://github.com/lus/pasty.git synced 2023-08-10 21:13:09 +03:00

Implement automatic paste deletion

This commit is contained in:
Lukas SP 2020-09-19 01:56:50 +02:00
parent 17d2fa91c5
commit 4048edbabb
9 changed files with 180 additions and 3 deletions

View File

@ -11,9 +11,17 @@ Pasty is a fast and lightweight code pasting server
| `PASTY_DELETION_TOKEN_LENGTH` | `12` | `number` | Defines the length of the deletion token of a paste | | `PASTY_DELETION_TOKEN_LENGTH` | `12` | `number` | Defines the length of the deletion token of a paste |
| `PASTY_RATE_LIMIT` | `30-M` | `string` | Defines the rate limit of the API (see https://github.com/ulule/limiter#usage) | | `PASTY_RATE_LIMIT` | `30-M` | `string` | Defines the rate limit of the API (see https://github.com/ulule/limiter#usage) |
## AutoDelete
Pasty provides an intuitive system to automatically delete pastes after a specific amount of time. You can configure it with the following variables:
## Storage types ## Storage types
Pasty supports multiple storage types, defined using the `PASTY_STORAGE_TYPE` environment variable (use the value behind the corresponding title in this README). Pasty supports multiple storage types, defined using the `PASTY_STORAGE_TYPE` environment variable (use the value behind the corresponding title in this README).
Every single one of them has its own configuration variables: Every single one of them has its own configuration variables:
| Environment Variable | Default Value | Type | Description |
|----------------------------------|---------------|----------|--------------------------------------------------------------------------------|
| `PASTY_AUTODELETE` | `false` | `bool` | Defines whether or not the AutoDelete system should be enabled |
| `PASTY_AUTODELETE_LIFETIME` | `720h` | `string` | Defines the duration a paste should live until it gets deleted |
| `PASTY_AUTODELETE_TASK_INTERVAL` | `5m` | `string` | Defines the interval in which the AutoDelete task should clean up the database |
### File (`file`) ### File (`file`)
| Environment Variable | Default Value | Type | Description | | Environment Variable | Default Value | Type | Description |

View File

@ -5,6 +5,7 @@ import (
"github.com/Lukaesebrot/pasty/internal/storage" "github.com/Lukaesebrot/pasty/internal/storage"
"github.com/Lukaesebrot/pasty/internal/web" "github.com/Lukaesebrot/pasty/internal/web"
"log" "log"
"time"
) )
func main() { func main() {
@ -26,6 +27,24 @@ func main() {
} }
}() }()
// Schedule the AutoDelete task
if env.Bool("AUTODELETE", false) {
log.Println("Scheduling the AutoDelete task...")
go func() {
for {
// Run the cleanup sequence
deleted, err := storage.Current.Cleanup()
if err != nil {
log.Fatalln(err)
}
log.Printf("AutoDelete: Deleted %d expired pastes", deleted)
// Wait until the process should repeat
time.Sleep(env.Duration("AUTODELETE_TASK_INTERVAL", 5*time.Minute))
}
}()
}
// Serve the web resources // Serve the web resources
log.Println("Serving the web resources...") log.Println("Serving the web resources...")
panic(web.Serve()) panic(web.Serve())

7
internal/env/env.go vendored
View File

@ -5,6 +5,7 @@ import (
"github.com/joho/godotenv" "github.com/joho/godotenv"
"os" "os"
"strconv" "strconv"
"time"
) )
// Load loads an optional .env file // Load loads an optional .env file
@ -26,3 +27,9 @@ func Bool(key string, fallback bool) bool {
parsed, _ := strconv.ParseBool(Get(key, strconv.FormatBool(fallback))) parsed, _ := strconv.ParseBool(Get(key, strconv.FormatBool(fallback)))
return parsed return parsed
} }
// Duration uses Get and parses it into a duration
func Duration(key string, fallback time.Duration) time.Duration {
parsed, _ := time.ParseDuration(Get(key, fallback.String()))
return parsed
}

View File

@ -1,7 +1,9 @@
package pastes package pastes
import ( import (
"github.com/Lukaesebrot/pasty/internal/env"
"github.com/alexedwards/argon2id" "github.com/alexedwards/argon2id"
"time"
) )
// Paste represents a saved paste // Paste represents a saved paste
@ -10,6 +12,8 @@ type Paste struct {
Content string `json:"content" bson:"content"` Content string `json:"content" bson:"content"`
SuggestedSyntaxType string `json:"suggestedSyntaxType" bson:"suggestedSyntaxType"` SuggestedSyntaxType string `json:"suggestedSyntaxType" bson:"suggestedSyntaxType"`
DeletionToken string `json:"deletionToken" bson:"deletionToken"` DeletionToken string `json:"deletionToken" bson:"deletionToken"`
Created int64 `json:"created" bson:"created"`
AutoDelete bool `json:"autoDelete" bson:"autoDelete"`
} }
// Create creates a new paste object using the given content // Create creates a new paste object using the given content
@ -29,6 +33,8 @@ func Create(id, content string) (*Paste, error) {
Content: content, Content: content,
SuggestedSyntaxType: suggestedSyntaxType, SuggestedSyntaxType: suggestedSyntaxType,
DeletionToken: deletionToken, DeletionToken: deletionToken,
Created: time.Now().Unix(),
AutoDelete: env.Bool("AUTODELETE", false),
}, nil }, nil
} }

View File

@ -18,6 +18,7 @@ type Driver interface {
Get(id string) (*pastes.Paste, error) Get(id string) (*pastes.Paste, error)
Save(paste *pastes.Paste) error Save(paste *pastes.Paste) error
Delete(id string) error Delete(id string) error
Cleanup() (int, error)
} }
// Load loads the current storage driver // Load loads the current storage driver

View File

@ -9,6 +9,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time"
) )
// FileDriver represents the file storage driver // FileDriver represents the file storage driver
@ -104,3 +105,35 @@ func (driver *FileDriver) Delete(id string) error {
id = base64.StdEncoding.EncodeToString([]byte(id)) id = base64.StdEncoding.EncodeToString([]byte(id))
return os.Remove(filepath.Join(driver.filePath, id+".json")) return os.Remove(filepath.Join(driver.filePath, id+".json"))
} }
// Cleanup cleans up the expired pastes
func (driver *FileDriver) Cleanup() (int, error) {
// Retrieve all paste IDs
ids, err := driver.ListIDs()
if err != nil {
return 0, err
}
// Define the amount of deleted items
deleted := 0
// Loop through all pastes
for _, id := range ids {
// Retrieve the paste object
paste, err := driver.Get(id)
if err != nil {
return 0, err
}
// Delete the paste if it is expired
lifetime := env.Duration("AUTODELETE_LIFETIME", 30*24*time.Hour)
if paste.AutoDelete && paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() {
err = driver.Delete(id)
if err != nil {
return 0, err
}
deleted++
}
}
return deleted, nil
}

View File

@ -101,7 +101,10 @@ func (driver *MongoDBDriver) Get(id string) (*pastes.Paste, error) {
// Return the retrieved paste object // Return the retrieved paste object
paste := new(pastes.Paste) paste := new(pastes.Paste)
err = result.Decode(paste) err = result.Decode(paste)
return paste, err if err != nil {
return nil, err
}
return paste, nil
} }
// Save saves a paste // Save saves a paste
@ -132,3 +135,35 @@ func (driver *MongoDBDriver) Delete(id string) error {
_, err := collection.DeleteOne(ctx, filter) _, err := collection.DeleteOne(ctx, filter)
return err return err
} }
// Cleanup cleans up the expired pastes
func (driver *MongoDBDriver) Cleanup() (int, error) {
// Retrieve all paste IDs
ids, err := driver.ListIDs()
if err != nil {
return 0, err
}
// Define the amount of deleted items
deleted := 0
// Loop through all pastes
for _, id := range ids {
// Retrieve the paste object
paste, err := driver.Get(id)
if err != nil {
return 0, err
}
// Delete the paste if it is expired
lifetime := env.Duration("AUTODELETE_LIFETIME", 30*24*time.Hour)
if paste.AutoDelete && paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() {
err = driver.Delete(id)
if err != nil {
return 0, err
}
deleted++
}
}
return deleted, nil
}

View File

@ -10,6 +10,7 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/credentials"
"io/ioutil" "io/ioutil"
"strings" "strings"
"time"
) )
// S3Driver represents the AWS S3 storage driver // S3Driver represents the AWS S3 storage driver
@ -100,3 +101,35 @@ func (driver *S3Driver) Save(paste *pastes.Paste) error {
func (driver *S3Driver) Delete(id string) error { func (driver *S3Driver) Delete(id string) error {
return driver.client.RemoveObject(context.Background(), driver.bucket, id+".json", minio.RemoveObjectOptions{}) return driver.client.RemoveObject(context.Background(), driver.bucket, id+".json", minio.RemoveObjectOptions{})
} }
// Cleanup cleans up the expired pastes
func (driver *S3Driver) Cleanup() (int, error) {
// Retrieve all paste IDs
ids, err := driver.ListIDs()
if err != nil {
return 0, err
}
// Define the amount of deleted items
deleted := 0
// Loop through all pastes
for _, id := range ids {
// Retrieve the paste object
paste, err := driver.Get(id)
if err != nil {
return 0, err
}
// Delete the paste if it is expired
lifetime := env.Duration("AUTODELETE_LIFETIME", 30*24*time.Hour)
if paste.AutoDelete && paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() {
err = driver.Delete(id)
if err != nil {
return 0, err
}
deleted++
}
}
return deleted, nil
}

View File

@ -7,6 +7,7 @@ import (
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq" _ "github.com/lib/pq"
_ "github.com/mattn/go-sqlite3" _ "github.com/mattn/go-sqlite3"
"time"
) )
// SQLDriver represents the SQL storage driver // SQLDriver represents the SQL storage driver
@ -36,7 +37,9 @@ func (driver *SQLDriver) Initialize() error {
id varchar NOT NULL PRIMARY KEY, id varchar NOT NULL PRIMARY KEY,
content varchar NOT NULL, content varchar NOT NULL,
suggestedSyntaxType varchar NOT NULL, suggestedSyntaxType varchar NOT NULL,
deletionToken varchar NOT NULL deletionToken varchar NOT NULL,
created bigint NOT NULL,
autoDelete bool NOT NULL
); );
`, table) `, table)
if err != nil { if err != nil {
@ -96,7 +99,7 @@ func (driver *SQLDriver) Get(id string) (*pastes.Paste, error) {
// Save saves a paste // Save saves a paste
func (driver *SQLDriver) Save(paste *pastes.Paste) error { func (driver *SQLDriver) Save(paste *pastes.Paste) error {
// Execute an INSERT statement to create the paste // Execute an INSERT statement to create the paste
_, err := driver.database.Exec("INSERT INTO ? (?, ?, ?, ?)", driver.table, paste.ID, paste.Content, paste.SuggestedSyntaxType, paste.DeletionToken) _, err := driver.database.Exec("INSERT INTO ? (?, ?, ?, ?, ?, ?)", driver.table, paste.ID, paste.Content, paste.SuggestedSyntaxType, paste.DeletionToken, paste.Created, paste.AutoDelete)
return err return err
} }
@ -106,3 +109,35 @@ func (driver *SQLDriver) Delete(id string) error {
_, err := driver.database.Exec("DELETE FROM ? WHERE id = ?", driver.table, id) _, err := driver.database.Exec("DELETE FROM ? WHERE id = ?", driver.table, id)
return err return err
} }
// Cleanup cleans up the expired pastes
func (driver *SQLDriver) Cleanup() (int, error) {
// Retrieve all paste IDs
ids, err := driver.ListIDs()
if err != nil {
return 0, err
}
// Define the amount of deleted items
deleted := 0
// Loop through all pastes
for _, id := range ids {
// Retrieve the paste object
paste, err := driver.Get(id)
if err != nil {
return 0, err
}
// Delete the paste if it is expired
lifetime := env.Duration("AUTODELETE_LIFETIME", 30*24*time.Hour)
if paste.AutoDelete && paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() {
err = driver.Delete(id)
if err != nil {
return 0, err
}
deleted++
}
}
return deleted, nil
}