SELU activation and yolo openimages

cfg/openimages.data

@@ -0,0 +1,8 @@
+classes= 601
+train  = /home/pjreddie/data/openimsv4/openimages.train.list
+#valid  = coco_testdev
+valid = data/coco_val_5k.list
+names = data/openimages.names
+backup = /home/pjreddie/backup/
+eval=coco
+

cfg/resnext101-32x4d.cfg

@@ -1,21 +1,25 @@
 [net]
-batch=128
+# Training
-subdivisions=8
+# batch=128
- batch=1
+# subdivisions=8
- subdivisions=1
+
+# Testing
+batch=1
+subdivisions=1
+
 height=256
 width=256
+channels=3
 min_crop=128
 max_crop=448
-channels=3
-momentum=0.9
-decay=0.0005
 
 burn_in=1000
 learning_rate=0.1
 policy=poly
 power=4
 max_batches=800000
+momentum=0.9
+decay=0.0005
 
 angle=7
 hue=.1
@@ -24,6 +28,7 @@ exposure=.75
 aspect=.75
 
 
+
 [convolutional]
 batch_normalize=1
 filters=64

cfg/resnext152-32x4d.cfg

@@ -1,21 +1,25 @@
 [net]
-batch=128
+# Training
-subdivisions=16
+# batch=128
-# batch=1
+# subdivisions=16
-# subdivisions=1
+
+# Testing
+batch=1
+subdivisions=1
+
 height=256
 width=256
+channels=3
 min_crop=128
 max_crop=448
-channels=3
-momentum=0.9
-decay=0.0005
 
 burn_in=1000
 learning_rate=0.1
 policy=poly
 power=4
 max_batches=800000
+momentum=0.9
+decay=0.0005
 
 angle=7
 hue=.1

cfg/vgg-16.cfg

@@ -1,6 +1,12 @@
 [net]
-batch=128
+# Training
-subdivisions=4
+# batch=128
+# subdivisions=4
+
+# Testing
+batch=1
+subdivisions=1
+
 height=256
 width=256
 channels=3

cfg/yolov3-openimages.cfg

@@ -0,0 +1,789 @@
+[net]
+# Testing
+ batch=1
+ subdivisions=1
+# Training
+batch=64
+subdivisions=16
+width=608
+height=608
+channels=3
+momentum=0.9
+decay=0.0005
+angle=0
+saturation = 1.5
+exposure = 1.5
+hue=.1
+
+learning_rate=0.001
+burn_in=5000
+max_batches = 500200
+policy=steps
+steps=400000,450000
+scales=.1,.1
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=3
+stride=1
+pad=1
+activation=leaky
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=32
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=64
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+# Downsample
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=2
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=1024
+size=3
+stride=1
+pad=1
+activation=leaky
+
+[shortcut]
+from=-3
+activation=linear
+
+######################
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=512
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=1024
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=1818
+activation=linear
+
+
+[yolo]
+mask = 6,7,8
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=601
+num=9
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 61
+
+
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=256
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=512
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=1818
+activation=linear
+
+
+[yolo]
+mask = 3,4,5
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=601
+num=9
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+
+
+
+[route]
+layers = -4
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[upsample]
+stride=2
+
+[route]
+layers = -1, 36
+
+
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+filters=128
+size=1
+stride=1
+pad=1
+activation=leaky
+
+[convolutional]
+batch_normalize=1
+size=3
+stride=1
+pad=1
+filters=256
+activation=leaky
+
+[convolutional]
+size=1
+stride=1
+pad=1
+filters=1818
+activation=linear
+
+
+[yolo]
+mask = 0,1,2
+anchors = 10,13,  16,30,  33,23,  30,61,  62,45,  59,119,  116,90,  156,198,  373,326
+classes=601
+num=9
+jitter=.3
+ignore_thresh = .7
+truth_thresh = 1
+random=1
+

data/openimages.names

@@ -0,0 +1,601 @@
+Tortoise
+Container
+Magpie
+Sea turtle
+Football
+Ambulance
+Ladder
+Toothbrush
+Syringe
+Sink
+Toy
+Organ
+Cassette deck
+Apple
+Human eye
+Cosmetics
+Paddle
+Snowman
+Beer
+Chopsticks
+Human beard
+Bird
+Parking meter
+Traffic light
+Croissant
+Cucumber
+Radish
+Towel
+Doll
+Skull
+Washing machine
+Glove
+Tick
+Belt
+Sunglasses
+Banjo
+Cart
+Ball
+Backpack
+Bicycle
+Home appliance
+Centipede
+Boat
+Surfboard
+Boot
+Headphones
+Hot dog
+Shorts
+Fast food
+Bus
+Boy
+Screwdriver
+Bicycle wheel
+Barge
+Laptop
+Miniskirt
+Drill
+Dress
+Bear
+Waffle
+Pancake
+Brown bear
+Woodpecker
+Blue jay
+Pretzel
+Bagel
+Tower
+Teapot
+Person
+Bow and arrow
+Swimwear
+Beehive
+Brassiere
+Bee
+Bat
+Starfish
+Popcorn
+Burrito
+Chainsaw
+Balloon
+Wrench
+Tent
+Vehicle registration plate
+Lantern
+Toaster
+Flashlight
+Billboard
+Tiara
+Limousine
+Necklace
+Carnivore
+Scissors
+Stairs
+Computer keyboard
+Printer
+Traffic sign
+Chair
+Shirt
+Poster
+Cheese
+Sock
+Fire hydrant
+Land vehicle
+Earrings
+Tie
+Watercraft
+Cabinetry
+Suitcase
+Muffin
+Bidet
+Snack
+Snowmobile
+Clock
+Medical equipment
+Cattle
+Cello
+Jet ski
+Camel
+Coat
+Suit
+Desk
+Cat
+Bronze sculpture
+Juice
+Gondola
+Beetle
+Cannon
+Computer mouse
+Cookie
+Office building
+Fountain
+Coin
+Calculator
+Cocktail
+Computer monitor
+Box
+Stapler
+Christmas tree
+Cowboy hat
+Hiking equipment
+Studio couch
+Drum
+Dessert
+Wine rack
+Drink
+Zucchini
+Ladle
+Human mouth
+Dairy
+Dice
+Oven
+Dinosaur
+Ratchet
+Couch
+Cricket ball
+Winter melon
+Spatula
+Whiteboard
+Pencil sharpener
+Door
+Hat
+Shower
+Eraser
+Fedora
+Guacamole
+Dagger
+Scarf
+Dolphin
+Sombrero
+Tin can
+Mug
+Tap
+Harbor seal
+Stretcher
+Can opener
+Goggles
+Human body
+Roller skates
+Coffee cup
+Cutting board
+Blender
+Plumbing fixture
+Stop sign
+Office supplies
+Volleyball
+Vase
+Slow cooker
+Wardrobe
+Coffee
+Whisk
+Paper towel
+Personal care
+Food
+Sun hat
+Tree house
+Flying disc
+Skirt
+Gas stove
+Salt and pepper shakers
+Mechanical fan
+Face powder
+Fax
+Fruit
+French fries
+Nightstand
+Barrel
+Kite
+Tart
+Treadmill
+Fox
+Flag
+Horn
+Window blind
+Human foot
+Golf cart
+Jacket
+Egg
+Street light
+Guitar
+Pillow
+Human leg
+Isopod
+Grape
+Human ear
+Power plugs and sockets
+Panda
+Giraffe
+Woman
+Door handle
+Rhinoceros
+Bathtub
+Goldfish
+Houseplant
+Goat
+Baseball bat
+Baseball glove
+Mixing bowl
+Marine invertebrates
+Kitchen utensil
+Light switch
+House
+Horse
+Stationary bicycle
+Hammer
+Ceiling fan
+Sofa bed
+Adhesive tape
+Harp
+Sandal
+Bicycle helmet
+Saucer
+Harpsichord
+Human hair
+Heater
+Harmonica
+Hamster
+Curtain
+Bed
+Kettle
+Fireplace
+Scale
+Drinking straw
+Insect
+Hair dryer
+Kitchenware
+Indoor rower
+Invertebrate
+Food processor
+Bookcase
+Refrigerator
+Wood-burning stove
+Punching bag
+Common fig
+Cocktail shaker
+Jaguar
+Golf ball
+Fashion accessory
+Alarm clock
+Filing cabinet
+Artichoke
+Table
+Tableware
+Kangaroo
+Koala
+Knife
+Bottle
+Bottle opener
+Lynx
+Lavender
+Lighthouse
+Dumbbell
+Human head
+Bowl
+Humidifier
+Porch
+Lizard
+Billiard table
+Mammal
+Mouse
+Motorcycle
+Musical instrument
+Swim cap
+Frying pan
+Snowplow
+Bathroom cabinet
+Missile
+Bust
+Man
+Waffle iron
+Milk
+Ring binder
+Plate
+Mobile phone
+Baked goods
+Mushroom
+Crutch
+Pitcher
+Mirror
+Lifejacket
+Table tennis racket
+Pencil case
+Musical keyboard
+Scoreboard
+Briefcase
+Kitchen knife
+Nail
+Tennis ball
+Plastic bag
+Oboe
+Chest of drawers
+Ostrich
+Piano
+Girl
+Plant
+Potato
+Hair spray
+Sports equipment
+Pasta
+Penguin
+Pumpkin
+Pear
+Infant bed
+Polar bear
+Mixer
+Cupboard
+Jacuzzi
+Pizza
+Digital clock
+Pig
+Reptile
+Rifle
+Lipstick
+Skateboard
+Raven
+High heels
+Red panda
+Rose
+Rabbit
+Sculpture
+Saxophone
+Shotgun
+Seafood
+Submarine sandwich
+Snowboard
+Sword
+Picture frame
+Sushi
+Loveseat
+Ski
+Squirrel
+Tripod
+Stethoscope
+Submarine
+Scorpion
+Segway
+Training bench
+Snake
+Coffee table
+Skyscraper
+Sheep
+Television
+Trombone
+Tea
+Tank
+Taco
+Telephone
+Torch
+Tiger
+Strawberry
+Trumpet
+Tree
+Tomato
+Train
+Tool
+Picnic basket
+Cooking spray
+Trousers
+Bowling equipment
+Football helmet
+Truck
+Measuring cup
+Coffeemaker
+Violin
+Vehicle
+Handbag
+Paper cutter
+Wine
+Weapon
+Wheel
+Worm
+Wok
+Whale
+Zebra
+Auto part
+Jug
+Pizza cutter
+Cream
+Monkey
+Lion
+Bread
+Platter
+Chicken
+Eagle
+Helicopter
+Owl
+Duck
+Turtle
+Hippopotamus
+Crocodile
+Toilet
+Toilet paper
+Squid
+Clothing
+Footwear
+Lemon
+Spider
+Deer
+Frog
+Banana
+Rocket
+Wine glass
+Countertop
+Tablet computer
+Waste container
+Swimming pool
+Dog
+Book
+Elephant
+Shark
+Candle
+Leopard
+Axe
+Hand dryer
+Soap dispenser
+Porcupine
+Flower
+Canary
+Cheetah
+Palm tree
+Hamburger
+Maple
+Building
+Fish
+Lobster
+Asparagus
+Furniture
+Hedgehog
+Airplane
+Spoon
+Otter
+Bull
+Oyster
+Horizontal bar
+Convenience store
+Bomb
+Bench
+Ice cream
+Caterpillar
+Butterfly
+Parachute
+Orange
+Antelope
+Beaker
+Moths and butterflies
+Window
+Closet
+Castle
+Jellyfish
+Goose
+Mule
+Swan
+Peach
+Coconut
+Seat belt
+Raccoon
+Chisel
+Fork
+Lamp
+Camera
+Squash
+Racket
+Human face
+Human arm
+Vegetable
+Diaper
+Unicycle
+Falcon
+Chime
+Snail
+Shellfish
+Cabbage
+Carrot
+Mango
+Jeans
+Flowerpot
+Pineapple
+Drawer
+Stool
+Envelope
+Cake
+Dragonfly
+Sunflower
+Microwave oven
+Honeycomb
+Marine mammal
+Sea lion
+Ladybug
+Shelf
+Watch
+Candy
+Salad
+Parrot
+Handgun
+Sparrow
+Van
+Grinder
+Spice rack
+Light bulb
+Corded phone
+Sports uniform
+Tennis racket
+Wall clock
+Serving tray
+Kitchen & dining room table
+Dog bed
+Cake stand
+Cat furniture
+Bathroom accessory
+Facial tissue holder
+Pressure cooker
+Kitchen appliance
+Tire
+Ruler
+Luggage and bags
+Microphone
+Broccoli
+Umbrella
+Pastry
+Grapefruit
+Band-aid
+Animal
+Bell pepper
+Turkey
+Lily
+Pomegranate
+Doughnut
+Glasses
+Human nose
+Pen
+Ant
+Car
+Aircraft
+Human hand
+Skunk
+Teddy bear
+Watermelon
+Cantaloupe
+Dishwasher
+Flute
+Balance beam
+Sandwich
+Shrimp
+Sewing machine
+Binoculars
+Rays and skates
+Ipod
+Accordion
+Willow
+Crab
+Crown
+Seahorse
+Perfume
+Alpaca
+Taxi
+Canoe
+Remote control
+Wheelchair
+Rugby ball
+Armadillo
+Maracas
+Helmet

include/darknet.h

@@ -54,7 +54,7 @@ typedef struct{
 tree *read_tree(char *filename);
 
 typedef enum{
-    LOGISTIC, RELU, RELIE, LINEAR, RAMP, TANH, PLSE, LEAKY, ELU, LOGGY, STAIR, HARDTAN, LHTAN
+    LOGISTIC, RELU, RELIE, LINEAR, RAMP, TANH, PLSE, LEAKY, ELU, LOGGY, STAIR, HARDTAN, LHTAN, SELU
 } ACTIVATION;
 
 typedef enum{

src/activation_kernels.cu

@@ -31,6 +31,7 @@ __device__ float logistic_activate_kernel(float x){return 1.f/(1.f + expf(-x));}
 __device__ float loggy_activate_kernel(float x){return 2.f/(1.f + expf(-x)) - 1;}
 __device__ float relu_activate_kernel(float x){return x*(x>0);}
 __device__ float elu_activate_kernel(float x){return (x >= 0)*x + (x < 0)*(expf(x)-1);}
+__device__ float selu_activate_kernel(float x){return (x >= 0)*1.0507f*x + (x < 0)*1.0507f*1.6732f*(expf(x)-1);}
 __device__ float relie_activate_kernel(float x){return (x>0) ? x : .01f*x;}
 __device__ float ramp_activate_kernel(float x){return x*(x>0)+.1f*x;}
 __device__ float leaky_activate_kernel(float x){return (x>0) ? x : .1f*x;}
@@ -63,6 +64,7 @@ __device__ float loggy_gradient_kernel(float x)
 }
 __device__ float relu_gradient_kernel(float x){return (x>0);}
 __device__ float elu_gradient_kernel(float x){return (x >= 0) + (x < 0)*(x + 1);}
+__device__ float selu_gradient_kernel(float x){return (x >= 0)*1.0507 + (x < 0)*(x + 1.0507*1.6732);}
 __device__ float relie_gradient_kernel(float x){return (x>0) ? 1 : .01f;}
 __device__ float ramp_gradient_kernel(float x){return (x>0)+.1f;}
 __device__ float leaky_gradient_kernel(float x){return (x>0) ? 1 : .1f;}
@@ -87,6 +89,8 @@ __device__ float activate_kernel(float x, ACTIVATION a)
             return relu_activate_kernel(x);
         case ELU:
             return elu_activate_kernel(x);
+        case SELU:
+            return selu_activate_kernel(x);
         case RELIE:
             return relie_activate_kernel(x);
         case RAMP:
@@ -120,6 +124,8 @@ __device__ float gradient_kernel(float x, ACTIVATION a)
             return relu_gradient_kernel(x);
         case ELU:
             return elu_gradient_kernel(x);
+        case SELU:
+            return selu_gradient_kernel(x);
         case RELIE:
             return relie_gradient_kernel(x);
         case RAMP:

src/activations.c

@@ -16,6 +16,8 @@ char *get_activation_string(ACTIVATION a)
             return "relu";
         case ELU:
             return "elu";
+        case SELU:
+            return "selu";
         case RELIE:
             return "relie";
         case RAMP:
@@ -46,6 +48,7 @@ ACTIVATION get_activation(char *s)
     if (strcmp(s, "loggy")==0) return LOGGY;
     if (strcmp(s, "relu")==0) return RELU;
     if (strcmp(s, "elu")==0) return ELU;
+    if (strcmp(s, "selu")==0) return SELU;
     if (strcmp(s, "relie")==0) return RELIE;
     if (strcmp(s, "plse")==0) return PLSE;
     if (strcmp(s, "hardtan")==0) return HARDTAN;
@@ -72,6 +75,8 @@ float activate(float x, ACTIVATION a)
             return relu_activate(x);
         case ELU:
             return elu_activate(x);
+        case SELU:
+            return selu_activate(x);
         case RELIE:
             return relie_activate(x);
         case RAMP:
@@ -113,6 +118,8 @@ float gradient(float x, ACTIVATION a)
             return relu_gradient(x);
         case ELU:
             return elu_gradient(x);
+        case SELU:
+            return selu_gradient(x);
         case RELIE:
             return relie_gradient(x);
         case RAMP:

src/activations.h

@@ -33,6 +33,7 @@ static inline float logistic_activate(float x){return 1./(1. + exp(-x));}
 static inline float loggy_activate(float x){return 2./(1. + exp(-x)) - 1;}
 static inline float relu_activate(float x){return x*(x>0);}
 static inline float elu_activate(float x){return (x >= 0)*x + (x < 0)*(exp(x)-1);}
+static inline float selu_activate(float x){return (x >= 0)*1.0507*x + (x < 0)*1.0507*1.6732*(exp(x)-1);}
 static inline float relie_activate(float x){return (x>0) ? x : .01*x;}
 static inline float ramp_activate(float x){return x*(x>0)+.1*x;}
 static inline float leaky_activate(float x){return (x>0) ? x : .1*x;}
@@ -75,6 +76,7 @@ static inline float stair_gradient(float x)
 }
 static inline float relu_gradient(float x){return (x>0);}
 static inline float elu_gradient(float x){return (x >= 0) + (x < 0)*(x + 1);}
+static inline float selu_gradient(float x){return (x >= 0)*1.0507 + (x < 0)*(x + 1.0507*1.6732);}
 static inline float relie_gradient(float x){return (x>0) ? 1 : .01;}
 static inline float ramp_gradient(float x){return (x>0)+.1;}
 static inline float leaky_gradient(float x){return (x>0) ? 1 : .1;}