diff --git a/cfg/openimages.data b/cfg/openimages.data new file mode 100644 index 00000000..fa80e5ab --- /dev/null +++ b/cfg/openimages.data @@ -0,0 +1,8 @@ +classes= 601 +train = /home/pjreddie/data/openimsv4/openimages.train.list +#valid = coco_testdev +valid = data/coco_val_5k.list +names = data/openimages.names +backup = /home/pjreddie/backup/ +eval=coco + diff --git a/cfg/resnext101-32x4d.cfg b/cfg/resnext101-32x4d.cfg index 382836d1..8538ccc3 100644 --- a/cfg/resnext101-32x4d.cfg +++ b/cfg/resnext101-32x4d.cfg @@ -1,21 +1,25 @@ [net] -batch=128 -subdivisions=8 - batch=1 - subdivisions=1 +# Training +# batch=128 +# subdivisions=8 + +# Testing +batch=1 +subdivisions=1 + height=256 width=256 +channels=3 min_crop=128 max_crop=448 -channels=3 -momentum=0.9 -decay=0.0005 burn_in=1000 learning_rate=0.1 policy=poly power=4 max_batches=800000 +momentum=0.9 +decay=0.0005 angle=7 hue=.1 @@ -24,6 +28,7 @@ exposure=.75 aspect=.75 + [convolutional] batch_normalize=1 filters=64 diff --git a/cfg/resnext152-32x4d.cfg b/cfg/resnext152-32x4d.cfg index 1296cf9c..48279fd2 100644 --- a/cfg/resnext152-32x4d.cfg +++ b/cfg/resnext152-32x4d.cfg @@ -1,21 +1,25 @@ [net] -batch=128 -subdivisions=16 -# batch=1 -# subdivisions=1 +# Training +# batch=128 +# subdivisions=16 + +# Testing +batch=1 +subdivisions=1 + height=256 width=256 +channels=3 min_crop=128 max_crop=448 -channels=3 -momentum=0.9 -decay=0.0005 burn_in=1000 learning_rate=0.1 policy=poly power=4 max_batches=800000 +momentum=0.9 +decay=0.0005 angle=7 hue=.1 diff --git a/cfg/vgg-16.cfg b/cfg/vgg-16.cfg index 79694b87..c73b17b6 100644 --- a/cfg/vgg-16.cfg +++ b/cfg/vgg-16.cfg @@ -1,6 +1,12 @@ [net] -batch=128 -subdivisions=4 +# Training +# batch=128 +# subdivisions=4 + +# Testing +batch=1 +subdivisions=1 + height=256 width=256 channels=3 diff --git a/cfg/yolov3-openimages.cfg b/cfg/yolov3-openimages.cfg new file mode 100644 index 00000000..65d241a7 --- /dev/null +++ b/cfg/yolov3-openimages.cfg @@ -0,0 +1,789 @@ +[net] +# Testing + batch=1 + subdivisions=1 +# Training +batch=64 +subdivisions=16 +width=608 +height=608 +channels=3 +momentum=0.9 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.001 +burn_in=5000 +max_batches = 500200 +policy=steps +steps=400000,450000 +scales=.1,.1 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=leaky + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +###################### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=1818 +activation=linear + + +[yolo] +mask = 6,7,8 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=601 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = -1, 61 + + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=1818 +activation=linear + + +[yolo] +mask = 3,4,5 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=601 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 + + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = -1, 36 + + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=1818 +activation=linear + + +[yolo] +mask = 0,1,2 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=601 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 + diff --git a/data/openimages.names b/data/openimages.names new file mode 100644 index 00000000..ddfd8f22 --- /dev/null +++ b/data/openimages.names @@ -0,0 +1,601 @@ +Tortoise +Container +Magpie +Sea turtle +Football +Ambulance +Ladder +Toothbrush +Syringe +Sink +Toy +Organ +Cassette deck +Apple +Human eye +Cosmetics +Paddle +Snowman +Beer +Chopsticks +Human beard +Bird +Parking meter +Traffic light +Croissant +Cucumber +Radish +Towel +Doll +Skull +Washing machine +Glove +Tick +Belt +Sunglasses +Banjo +Cart +Ball +Backpack +Bicycle +Home appliance +Centipede +Boat +Surfboard +Boot +Headphones +Hot dog +Shorts +Fast food +Bus +Boy +Screwdriver +Bicycle wheel +Barge +Laptop +Miniskirt +Drill +Dress +Bear +Waffle +Pancake +Brown bear +Woodpecker +Blue jay +Pretzel +Bagel +Tower +Teapot +Person +Bow and arrow +Swimwear +Beehive +Brassiere +Bee +Bat +Starfish +Popcorn +Burrito +Chainsaw +Balloon +Wrench +Tent +Vehicle registration plate +Lantern +Toaster +Flashlight +Billboard +Tiara +Limousine +Necklace +Carnivore +Scissors +Stairs +Computer keyboard +Printer +Traffic sign +Chair +Shirt +Poster +Cheese +Sock +Fire hydrant +Land vehicle +Earrings +Tie +Watercraft +Cabinetry +Suitcase +Muffin +Bidet +Snack +Snowmobile +Clock +Medical equipment +Cattle +Cello +Jet ski +Camel +Coat +Suit +Desk +Cat +Bronze sculpture +Juice +Gondola +Beetle +Cannon +Computer mouse +Cookie +Office building +Fountain +Coin +Calculator +Cocktail +Computer monitor +Box +Stapler +Christmas tree +Cowboy hat +Hiking equipment +Studio couch +Drum +Dessert +Wine rack +Drink +Zucchini +Ladle +Human mouth +Dairy +Dice +Oven +Dinosaur +Ratchet +Couch +Cricket ball +Winter melon +Spatula +Whiteboard +Pencil sharpener +Door +Hat +Shower +Eraser +Fedora +Guacamole +Dagger +Scarf +Dolphin +Sombrero +Tin can +Mug +Tap +Harbor seal +Stretcher +Can opener +Goggles +Human body +Roller skates +Coffee cup +Cutting board +Blender +Plumbing fixture +Stop sign +Office supplies +Volleyball +Vase +Slow cooker +Wardrobe +Coffee +Whisk +Paper towel +Personal care +Food +Sun hat +Tree house +Flying disc +Skirt +Gas stove +Salt and pepper shakers +Mechanical fan +Face powder +Fax +Fruit +French fries +Nightstand +Barrel +Kite +Tart +Treadmill +Fox +Flag +Horn +Window blind +Human foot +Golf cart +Jacket +Egg +Street light +Guitar +Pillow +Human leg +Isopod +Grape +Human ear +Power plugs and sockets +Panda +Giraffe +Woman +Door handle +Rhinoceros +Bathtub +Goldfish +Houseplant +Goat +Baseball bat +Baseball glove +Mixing bowl +Marine invertebrates +Kitchen utensil +Light switch +House +Horse +Stationary bicycle +Hammer +Ceiling fan +Sofa bed +Adhesive tape +Harp +Sandal +Bicycle helmet +Saucer +Harpsichord +Human hair +Heater +Harmonica +Hamster +Curtain +Bed +Kettle +Fireplace +Scale +Drinking straw +Insect +Hair dryer +Kitchenware +Indoor rower +Invertebrate +Food processor +Bookcase +Refrigerator +Wood-burning stove +Punching bag +Common fig +Cocktail shaker +Jaguar +Golf ball +Fashion accessory +Alarm clock +Filing cabinet +Artichoke +Table +Tableware +Kangaroo +Koala +Knife +Bottle +Bottle opener +Lynx +Lavender +Lighthouse +Dumbbell +Human head +Bowl +Humidifier +Porch +Lizard +Billiard table +Mammal +Mouse +Motorcycle +Musical instrument +Swim cap +Frying pan +Snowplow +Bathroom cabinet +Missile +Bust +Man +Waffle iron +Milk +Ring binder +Plate +Mobile phone +Baked goods +Mushroom +Crutch +Pitcher +Mirror +Lifejacket +Table tennis racket +Pencil case +Musical keyboard +Scoreboard +Briefcase +Kitchen knife +Nail +Tennis ball +Plastic bag +Oboe +Chest of drawers +Ostrich +Piano +Girl +Plant +Potato +Hair spray +Sports equipment +Pasta +Penguin +Pumpkin +Pear +Infant bed +Polar bear +Mixer +Cupboard +Jacuzzi +Pizza +Digital clock +Pig +Reptile +Rifle +Lipstick +Skateboard +Raven +High heels +Red panda +Rose +Rabbit +Sculpture +Saxophone +Shotgun +Seafood +Submarine sandwich +Snowboard +Sword +Picture frame +Sushi +Loveseat +Ski +Squirrel +Tripod +Stethoscope +Submarine +Scorpion +Segway +Training bench +Snake +Coffee table +Skyscraper +Sheep +Television +Trombone +Tea +Tank +Taco +Telephone +Torch +Tiger +Strawberry +Trumpet +Tree +Tomato +Train +Tool +Picnic basket +Cooking spray +Trousers +Bowling equipment +Football helmet +Truck +Measuring cup +Coffeemaker +Violin +Vehicle +Handbag +Paper cutter +Wine +Weapon +Wheel +Worm +Wok +Whale +Zebra +Auto part +Jug +Pizza cutter +Cream +Monkey +Lion +Bread +Platter +Chicken +Eagle +Helicopter +Owl +Duck +Turtle +Hippopotamus +Crocodile +Toilet +Toilet paper +Squid +Clothing +Footwear +Lemon +Spider +Deer +Frog +Banana +Rocket +Wine glass +Countertop +Tablet computer +Waste container +Swimming pool +Dog +Book +Elephant +Shark +Candle +Leopard +Axe +Hand dryer +Soap dispenser +Porcupine +Flower +Canary +Cheetah +Palm tree +Hamburger +Maple +Building +Fish +Lobster +Asparagus +Furniture +Hedgehog +Airplane +Spoon +Otter +Bull +Oyster +Horizontal bar +Convenience store +Bomb +Bench +Ice cream +Caterpillar +Butterfly +Parachute +Orange +Antelope +Beaker +Moths and butterflies +Window +Closet +Castle +Jellyfish +Goose +Mule +Swan +Peach +Coconut +Seat belt +Raccoon +Chisel +Fork +Lamp +Camera +Squash +Racket +Human face +Human arm +Vegetable +Diaper +Unicycle +Falcon +Chime +Snail +Shellfish +Cabbage +Carrot +Mango +Jeans +Flowerpot +Pineapple +Drawer +Stool +Envelope +Cake +Dragonfly +Sunflower +Microwave oven +Honeycomb +Marine mammal +Sea lion +Ladybug +Shelf +Watch +Candy +Salad +Parrot +Handgun +Sparrow +Van +Grinder +Spice rack +Light bulb +Corded phone +Sports uniform +Tennis racket +Wall clock +Serving tray +Kitchen & dining room table +Dog bed +Cake stand +Cat furniture +Bathroom accessory +Facial tissue holder +Pressure cooker +Kitchen appliance +Tire +Ruler +Luggage and bags +Microphone +Broccoli +Umbrella +Pastry +Grapefruit +Band-aid +Animal +Bell pepper +Turkey +Lily +Pomegranate +Doughnut +Glasses +Human nose +Pen +Ant +Car +Aircraft +Human hand +Skunk +Teddy bear +Watermelon +Cantaloupe +Dishwasher +Flute +Balance beam +Sandwich +Shrimp +Sewing machine +Binoculars +Rays and skates +Ipod +Accordion +Willow +Crab +Crown +Seahorse +Perfume +Alpaca +Taxi +Canoe +Remote control +Wheelchair +Rugby ball +Armadillo +Maracas +Helmet diff --git a/include/darknet.h b/include/darknet.h index abe4e5bf..b0a65e99 100644 --- a/include/darknet.h +++ b/include/darknet.h @@ -54,7 +54,7 @@ typedef struct{ tree *read_tree(char *filename); typedef enum{ - LOGISTIC, RELU, RELIE, LINEAR, RAMP, TANH, PLSE, LEAKY, ELU, LOGGY, STAIR, HARDTAN, LHTAN + LOGISTIC, RELU, RELIE, LINEAR, RAMP, TANH, PLSE, LEAKY, ELU, LOGGY, STAIR, HARDTAN, LHTAN, SELU } ACTIVATION; typedef enum{ diff --git a/src/activation_kernels.cu b/src/activation_kernels.cu index 5852eb58..4dc5804d 100644 --- a/src/activation_kernels.cu +++ b/src/activation_kernels.cu @@ -31,6 +31,7 @@ __device__ float logistic_activate_kernel(float x){return 1.f/(1.f + expf(-x));} __device__ float loggy_activate_kernel(float x){return 2.f/(1.f + expf(-x)) - 1;} __device__ float relu_activate_kernel(float x){return x*(x>0);} __device__ float elu_activate_kernel(float x){return (x >= 0)*x + (x < 0)*(expf(x)-1);} +__device__ float selu_activate_kernel(float x){return (x >= 0)*1.0507f*x + (x < 0)*1.0507f*1.6732f*(expf(x)-1);} __device__ float relie_activate_kernel(float x){return (x>0) ? x : .01f*x;} __device__ float ramp_activate_kernel(float x){return x*(x>0)+.1f*x;} __device__ float leaky_activate_kernel(float x){return (x>0) ? x : .1f*x;} @@ -63,6 +64,7 @@ __device__ float loggy_gradient_kernel(float x) } __device__ float relu_gradient_kernel(float x){return (x>0);} __device__ float elu_gradient_kernel(float x){return (x >= 0) + (x < 0)*(x + 1);} +__device__ float selu_gradient_kernel(float x){return (x >= 0)*1.0507 + (x < 0)*(x + 1.0507*1.6732);} __device__ float relie_gradient_kernel(float x){return (x>0) ? 1 : .01f;} __device__ float ramp_gradient_kernel(float x){return (x>0)+.1f;} __device__ float leaky_gradient_kernel(float x){return (x>0) ? 1 : .1f;} @@ -87,6 +89,8 @@ __device__ float activate_kernel(float x, ACTIVATION a) return relu_activate_kernel(x); case ELU: return elu_activate_kernel(x); + case SELU: + return selu_activate_kernel(x); case RELIE: return relie_activate_kernel(x); case RAMP: @@ -120,6 +124,8 @@ __device__ float gradient_kernel(float x, ACTIVATION a) return relu_gradient_kernel(x); case ELU: return elu_gradient_kernel(x); + case SELU: + return selu_gradient_kernel(x); case RELIE: return relie_gradient_kernel(x); case RAMP: diff --git a/src/activations.c b/src/activations.c index 0cbb2f55..da1a17a8 100644 --- a/src/activations.c +++ b/src/activations.c @@ -16,6 +16,8 @@ char *get_activation_string(ACTIVATION a) return "relu"; case ELU: return "elu"; + case SELU: + return "selu"; case RELIE: return "relie"; case RAMP: @@ -46,6 +48,7 @@ ACTIVATION get_activation(char *s) if (strcmp(s, "loggy")==0) return LOGGY; if (strcmp(s, "relu")==0) return RELU; if (strcmp(s, "elu")==0) return ELU; + if (strcmp(s, "selu")==0) return SELU; if (strcmp(s, "relie")==0) return RELIE; if (strcmp(s, "plse")==0) return PLSE; if (strcmp(s, "hardtan")==0) return HARDTAN; @@ -72,6 +75,8 @@ float activate(float x, ACTIVATION a) return relu_activate(x); case ELU: return elu_activate(x); + case SELU: + return selu_activate(x); case RELIE: return relie_activate(x); case RAMP: @@ -113,6 +118,8 @@ float gradient(float x, ACTIVATION a) return relu_gradient(x); case ELU: return elu_gradient(x); + case SELU: + return selu_gradient(x); case RELIE: return relie_gradient(x); case RAMP: diff --git a/src/activations.h b/src/activations.h index d456dbe3..9780d2cf 100644 --- a/src/activations.h +++ b/src/activations.h @@ -33,6 +33,7 @@ static inline float logistic_activate(float x){return 1./(1. + exp(-x));} static inline float loggy_activate(float x){return 2./(1. + exp(-x)) - 1;} static inline float relu_activate(float x){return x*(x>0);} static inline float elu_activate(float x){return (x >= 0)*x + (x < 0)*(exp(x)-1);} +static inline float selu_activate(float x){return (x >= 0)*1.0507*x + (x < 0)*1.0507*1.6732*(exp(x)-1);} static inline float relie_activate(float x){return (x>0) ? x : .01*x;} static inline float ramp_activate(float x){return x*(x>0)+.1*x;} static inline float leaky_activate(float x){return (x>0) ? x : .1*x;} @@ -75,6 +76,7 @@ static inline float stair_gradient(float x) } static inline float relu_gradient(float x){return (x>0);} static inline float elu_gradient(float x){return (x >= 0) + (x < 0)*(x + 1);} +static inline float selu_gradient(float x){return (x >= 0)*1.0507 + (x < 0)*(x + 1.0507*1.6732);} static inline float relie_gradient(float x){return (x>0) ? 1 : .01;} static inline float ramp_gradient(float x){return (x>0)+.1;} static inline float leaky_gradient(float x){return (x>0) ? 1 : .1;}