SELU activation and yolo openimages

This commit is contained in:
Joseph Redmon 2018-09-04 17:25:05 -07:00
parent 98e2a7674f
commit 680d3bde19
10 changed files with 1445 additions and 17 deletions

8
cfg/openimages.data Normal file
View File

@ -0,0 +1,8 @@
classes= 601
train = /home/pjreddie/data/openimsv4/openimages.train.list
#valid = coco_testdev
valid = data/coco_val_5k.list
names = data/openimages.names
backup = /home/pjreddie/backup/
eval=coco

View File

@ -1,21 +1,25 @@
[net]
batch=128
subdivisions=8
# Training
# batch=128
# subdivisions=8
# Testing
batch=1
subdivisions=1
height=256
width=256
channels=3
min_crop=128
max_crop=448
channels=3
momentum=0.9
decay=0.0005
burn_in=1000
learning_rate=0.1
policy=poly
power=4
max_batches=800000
momentum=0.9
decay=0.0005
angle=7
hue=.1
@ -24,6 +28,7 @@ exposure=.75
aspect=.75
[convolutional]
batch_normalize=1
filters=64

View File

@ -1,21 +1,25 @@
[net]
batch=128
subdivisions=16
# batch=1
# subdivisions=1
# Training
# batch=128
# subdivisions=16
# Testing
batch=1
subdivisions=1
height=256
width=256
channels=3
min_crop=128
max_crop=448
channels=3
momentum=0.9
decay=0.0005
burn_in=1000
learning_rate=0.1
policy=poly
power=4
max_batches=800000
momentum=0.9
decay=0.0005
angle=7
hue=.1

View File

@ -1,6 +1,12 @@
[net]
batch=128
subdivisions=4
# Training
# batch=128
# subdivisions=4
# Testing
batch=1
subdivisions=1
height=256
width=256
channels=3

789
cfg/yolov3-openimages.cfg Normal file
View File

@ -0,0 +1,789 @@
[net]
# Testing
batch=1
subdivisions=1
# Training
batch=64
subdivisions=16
width=608
height=608
channels=3
momentum=0.9
decay=0.0005
angle=0
saturation = 1.5
exposure = 1.5
hue=.1
learning_rate=0.001
burn_in=5000
max_batches = 500200
policy=steps
steps=400000,450000
scales=.1,.1
[convolutional]
batch_normalize=1
filters=32
size=3
stride=1
pad=1
activation=leaky
# Downsample
[convolutional]
batch_normalize=1
filters=64
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=32
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=64
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
# Downsample
[convolutional]
batch_normalize=1
filters=128
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=64
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=64
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
# Downsample
[convolutional]
batch_normalize=1
filters=256
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
# Downsample
[convolutional]
batch_normalize=1
filters=512
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
# Downsample
[convolutional]
batch_normalize=1
filters=1024
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=1024
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=1024
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=1024
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=1024
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
######################
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=1024
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=1024
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=1024
activation=leaky
[convolutional]
size=1
stride=1
pad=1
filters=1818
activation=linear
[yolo]
mask = 6,7,8
anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326
classes=601
num=9
jitter=.3
ignore_thresh = .7
truth_thresh = 1
random=1
[route]
layers = -4
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[upsample]
stride=2
[route]
layers = -1, 61
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=512
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=512
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=512
activation=leaky
[convolutional]
size=1
stride=1
pad=1
filters=1818
activation=linear
[yolo]
mask = 3,4,5
anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326
classes=601
num=9
jitter=.3
ignore_thresh = .7
truth_thresh = 1
random=1
[route]
layers = -4
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[upsample]
stride=2
[route]
layers = -1, 36
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=256
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=256
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=256
activation=leaky
[convolutional]
size=1
stride=1
pad=1
filters=1818
activation=linear
[yolo]
mask = 0,1,2
anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326
classes=601
num=9
jitter=.3
ignore_thresh = .7
truth_thresh = 1
random=1

601
data/openimages.names Normal file
View File

@ -0,0 +1,601 @@
Tortoise
Container
Magpie
Sea turtle
Football
Ambulance
Ladder
Toothbrush
Syringe
Sink
Toy
Organ
Cassette deck
Apple
Human eye
Cosmetics
Paddle
Snowman
Beer
Chopsticks
Human beard
Bird
Parking meter
Traffic light
Croissant
Cucumber
Radish
Towel
Doll
Skull
Washing machine
Glove
Tick
Belt
Sunglasses
Banjo
Cart
Ball
Backpack
Bicycle
Home appliance
Centipede
Boat
Surfboard
Boot
Headphones
Hot dog
Shorts
Fast food
Bus
Boy
Screwdriver
Bicycle wheel
Barge
Laptop
Miniskirt
Drill
Dress
Bear
Waffle
Pancake
Brown bear
Woodpecker
Blue jay
Pretzel
Bagel
Tower
Teapot
Person
Bow and arrow
Swimwear
Beehive
Brassiere
Bee
Bat
Starfish
Popcorn
Burrito
Chainsaw
Balloon
Wrench
Tent
Vehicle registration plate
Lantern
Toaster
Flashlight
Billboard
Tiara
Limousine
Necklace
Carnivore
Scissors
Stairs
Computer keyboard
Printer
Traffic sign
Chair
Shirt
Poster
Cheese
Sock
Fire hydrant
Land vehicle
Earrings
Tie
Watercraft
Cabinetry
Suitcase
Muffin
Bidet
Snack
Snowmobile
Clock
Medical equipment
Cattle
Cello
Jet ski
Camel
Coat
Suit
Desk
Cat
Bronze sculpture
Juice
Gondola
Beetle
Cannon
Computer mouse
Cookie
Office building
Fountain
Coin
Calculator
Cocktail
Computer monitor
Box
Stapler
Christmas tree
Cowboy hat
Hiking equipment
Studio couch
Drum
Dessert
Wine rack
Drink
Zucchini
Ladle
Human mouth
Dairy
Dice
Oven
Dinosaur
Ratchet
Couch
Cricket ball
Winter melon
Spatula
Whiteboard
Pencil sharpener
Door
Hat
Shower
Eraser
Fedora
Guacamole
Dagger
Scarf
Dolphin
Sombrero
Tin can
Mug
Tap
Harbor seal
Stretcher
Can opener
Goggles
Human body
Roller skates
Coffee cup
Cutting board
Blender
Plumbing fixture
Stop sign
Office supplies
Volleyball
Vase
Slow cooker
Wardrobe
Coffee
Whisk
Paper towel
Personal care
Food
Sun hat
Tree house
Flying disc
Skirt
Gas stove
Salt and pepper shakers
Mechanical fan
Face powder
Fax
Fruit
French fries
Nightstand
Barrel
Kite
Tart
Treadmill
Fox
Flag
Horn
Window blind
Human foot
Golf cart
Jacket
Egg
Street light
Guitar
Pillow
Human leg
Isopod
Grape
Human ear
Power plugs and sockets
Panda
Giraffe
Woman
Door handle
Rhinoceros
Bathtub
Goldfish
Houseplant
Goat
Baseball bat
Baseball glove
Mixing bowl
Marine invertebrates
Kitchen utensil
Light switch
House
Horse
Stationary bicycle
Hammer
Ceiling fan
Sofa bed
Adhesive tape
Harp
Sandal
Bicycle helmet
Saucer
Harpsichord
Human hair
Heater
Harmonica
Hamster
Curtain
Bed
Kettle
Fireplace
Scale
Drinking straw
Insect
Hair dryer
Kitchenware
Indoor rower
Invertebrate
Food processor
Bookcase
Refrigerator
Wood-burning stove
Punching bag
Common fig
Cocktail shaker
Jaguar
Golf ball
Fashion accessory
Alarm clock
Filing cabinet
Artichoke
Table
Tableware
Kangaroo
Koala
Knife
Bottle
Bottle opener
Lynx
Lavender
Lighthouse
Dumbbell
Human head
Bowl
Humidifier
Porch
Lizard
Billiard table
Mammal
Mouse
Motorcycle
Musical instrument
Swim cap
Frying pan
Snowplow
Bathroom cabinet
Missile
Bust
Man
Waffle iron
Milk
Ring binder
Plate
Mobile phone
Baked goods
Mushroom
Crutch
Pitcher
Mirror
Lifejacket
Table tennis racket
Pencil case
Musical keyboard
Scoreboard
Briefcase
Kitchen knife
Nail
Tennis ball
Plastic bag
Oboe
Chest of drawers
Ostrich
Piano
Girl
Plant
Potato
Hair spray
Sports equipment
Pasta
Penguin
Pumpkin
Pear
Infant bed
Polar bear
Mixer
Cupboard
Jacuzzi
Pizza
Digital clock
Pig
Reptile
Rifle
Lipstick
Skateboard
Raven
High heels
Red panda
Rose
Rabbit
Sculpture
Saxophone
Shotgun
Seafood
Submarine sandwich
Snowboard
Sword
Picture frame
Sushi
Loveseat
Ski
Squirrel
Tripod
Stethoscope
Submarine
Scorpion
Segway
Training bench
Snake
Coffee table
Skyscraper
Sheep
Television
Trombone
Tea
Tank
Taco
Telephone
Torch
Tiger
Strawberry
Trumpet
Tree
Tomato
Train
Tool
Picnic basket
Cooking spray
Trousers
Bowling equipment
Football helmet
Truck
Measuring cup
Coffeemaker
Violin
Vehicle
Handbag
Paper cutter
Wine
Weapon
Wheel
Worm
Wok
Whale
Zebra
Auto part
Jug
Pizza cutter
Cream
Monkey
Lion
Bread
Platter
Chicken
Eagle
Helicopter
Owl
Duck
Turtle
Hippopotamus
Crocodile
Toilet
Toilet paper
Squid
Clothing
Footwear
Lemon
Spider
Deer
Frog
Banana
Rocket
Wine glass
Countertop
Tablet computer
Waste container
Swimming pool
Dog
Book
Elephant
Shark
Candle
Leopard
Axe
Hand dryer
Soap dispenser
Porcupine
Flower
Canary
Cheetah
Palm tree
Hamburger
Maple
Building
Fish
Lobster
Asparagus
Furniture
Hedgehog
Airplane
Spoon
Otter
Bull
Oyster
Horizontal bar
Convenience store
Bomb
Bench
Ice cream
Caterpillar
Butterfly
Parachute
Orange
Antelope
Beaker
Moths and butterflies
Window
Closet
Castle
Jellyfish
Goose
Mule
Swan
Peach
Coconut
Seat belt
Raccoon
Chisel
Fork
Lamp
Camera
Squash
Racket
Human face
Human arm
Vegetable
Diaper
Unicycle
Falcon
Chime
Snail
Shellfish
Cabbage
Carrot
Mango
Jeans
Flowerpot
Pineapple
Drawer
Stool
Envelope
Cake
Dragonfly
Sunflower
Microwave oven
Honeycomb
Marine mammal
Sea lion
Ladybug
Shelf
Watch
Candy
Salad
Parrot
Handgun
Sparrow
Van
Grinder
Spice rack
Light bulb
Corded phone
Sports uniform
Tennis racket
Wall clock
Serving tray
Kitchen & dining room table
Dog bed
Cake stand
Cat furniture
Bathroom accessory
Facial tissue holder
Pressure cooker
Kitchen appliance
Tire
Ruler
Luggage and bags
Microphone
Broccoli
Umbrella
Pastry
Grapefruit
Band-aid
Animal
Bell pepper
Turkey
Lily
Pomegranate
Doughnut
Glasses
Human nose
Pen
Ant
Car
Aircraft
Human hand
Skunk
Teddy bear
Watermelon
Cantaloupe
Dishwasher
Flute
Balance beam
Sandwich
Shrimp
Sewing machine
Binoculars
Rays and skates
Ipod
Accordion
Willow
Crab
Crown
Seahorse
Perfume
Alpaca
Taxi
Canoe
Remote control
Wheelchair
Rugby ball
Armadillo
Maracas
Helmet

View File

@ -54,7 +54,7 @@ typedef struct{
tree *read_tree(char *filename);
typedef enum{
LOGISTIC, RELU, RELIE, LINEAR, RAMP, TANH, PLSE, LEAKY, ELU, LOGGY, STAIR, HARDTAN, LHTAN
LOGISTIC, RELU, RELIE, LINEAR, RAMP, TANH, PLSE, LEAKY, ELU, LOGGY, STAIR, HARDTAN, LHTAN, SELU
} ACTIVATION;
typedef enum{

View File

@ -31,6 +31,7 @@ __device__ float logistic_activate_kernel(float x){return 1.f/(1.f + expf(-x));}
__device__ float loggy_activate_kernel(float x){return 2.f/(1.f + expf(-x)) - 1;}
__device__ float relu_activate_kernel(float x){return x*(x>0);}
__device__ float elu_activate_kernel(float x){return (x >= 0)*x + (x < 0)*(expf(x)-1);}
__device__ float selu_activate_kernel(float x){return (x >= 0)*1.0507f*x + (x < 0)*1.0507f*1.6732f*(expf(x)-1);}
__device__ float relie_activate_kernel(float x){return (x>0) ? x : .01f*x;}
__device__ float ramp_activate_kernel(float x){return x*(x>0)+.1f*x;}
__device__ float leaky_activate_kernel(float x){return (x>0) ? x : .1f*x;}
@ -63,6 +64,7 @@ __device__ float loggy_gradient_kernel(float x)
}
__device__ float relu_gradient_kernel(float x){return (x>0);}
__device__ float elu_gradient_kernel(float x){return (x >= 0) + (x < 0)*(x + 1);}
__device__ float selu_gradient_kernel(float x){return (x >= 0)*1.0507 + (x < 0)*(x + 1.0507*1.6732);}
__device__ float relie_gradient_kernel(float x){return (x>0) ? 1 : .01f;}
__device__ float ramp_gradient_kernel(float x){return (x>0)+.1f;}
__device__ float leaky_gradient_kernel(float x){return (x>0) ? 1 : .1f;}
@ -87,6 +89,8 @@ __device__ float activate_kernel(float x, ACTIVATION a)
return relu_activate_kernel(x);
case ELU:
return elu_activate_kernel(x);
case SELU:
return selu_activate_kernel(x);
case RELIE:
return relie_activate_kernel(x);
case RAMP:
@ -120,6 +124,8 @@ __device__ float gradient_kernel(float x, ACTIVATION a)
return relu_gradient_kernel(x);
case ELU:
return elu_gradient_kernel(x);
case SELU:
return selu_gradient_kernel(x);
case RELIE:
return relie_gradient_kernel(x);
case RAMP:

View File

@ -16,6 +16,8 @@ char *get_activation_string(ACTIVATION a)
return "relu";
case ELU:
return "elu";
case SELU:
return "selu";
case RELIE:
return "relie";
case RAMP:
@ -46,6 +48,7 @@ ACTIVATION get_activation(char *s)
if (strcmp(s, "loggy")==0) return LOGGY;
if (strcmp(s, "relu")==0) return RELU;
if (strcmp(s, "elu")==0) return ELU;
if (strcmp(s, "selu")==0) return SELU;
if (strcmp(s, "relie")==0) return RELIE;
if (strcmp(s, "plse")==0) return PLSE;
if (strcmp(s, "hardtan")==0) return HARDTAN;
@ -72,6 +75,8 @@ float activate(float x, ACTIVATION a)
return relu_activate(x);
case ELU:
return elu_activate(x);
case SELU:
return selu_activate(x);
case RELIE:
return relie_activate(x);
case RAMP:
@ -113,6 +118,8 @@ float gradient(float x, ACTIVATION a)
return relu_gradient(x);
case ELU:
return elu_gradient(x);
case SELU:
return selu_gradient(x);
case RELIE:
return relie_gradient(x);
case RAMP:

View File

@ -33,6 +33,7 @@ static inline float logistic_activate(float x){return 1./(1. + exp(-x));}
static inline float loggy_activate(float x){return 2./(1. + exp(-x)) - 1;}
static inline float relu_activate(float x){return x*(x>0);}
static inline float elu_activate(float x){return (x >= 0)*x + (x < 0)*(exp(x)-1);}
static inline float selu_activate(float x){return (x >= 0)*1.0507*x + (x < 0)*1.0507*1.6732*(exp(x)-1);}
static inline float relie_activate(float x){return (x>0) ? x : .01*x;}
static inline float ramp_activate(float x){return x*(x>0)+.1*x;}
static inline float leaky_activate(float x){return (x>0) ? x : .1*x;}
@ -75,6 +76,7 @@ static inline float stair_gradient(float x)
}
static inline float relu_gradient(float x){return (x>0);}
static inline float elu_gradient(float x){return (x >= 0) + (x < 0)*(x + 1);}
static inline float selu_gradient(float x){return (x >= 0)*1.0507 + (x < 0)*(x + 1.0507*1.6732);}
static inline float relie_gradient(float x){return (x>0) ? 1 : .01;}
static inline float ramp_gradient(float x){return (x>0)+.1;}
static inline float leaky_gradient(float x){return (x>0) ? 1 : .1;}