added exponential, elu and softplus activation function#18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mnijh wants to merge 1 commit into
base:master
Choose a base branch
from
Open

added exponential, elu and softplus activation function #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions KerasWeightsProcessing/convert_weights.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -242,12 +242,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
)
)
# add information about the activation
layer_info.append(
info_str.format(
name = activation,
info = 0
if (activation == 'elu'):
layer_info.append(
info_str.format(
name = activation,
info = 1
)
)
)
else:
layer_info.append(
info_str.format(
name = activation,
info = 0
)
)

elif class_name == 'batchnormalization':
# get beta, gamma, moving_mean, moving_variance from dictionary
for key in sorted(model_weights[name][name].keys()):
Expand Down
59 changes: 59 additions & 0 deletions src/lib/mod_activation.F90
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,9 @@ module mod_activation
public :: tanhf, tanh_prime
public :: linear, linear_prime
public :: leaky_relu, leaky_relu_prime
public :: elu, elu_prime
public :: exponential, exponential_prime
public :: softplus, softplus_prime

interface
pure function activation_function(x, alpha)
Expand All@@ -28,6 +31,46 @@ end function activation_function

contains

pure function elu(x, alpha) result(res)
!! Exponential Linear Unit (ELU) activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
where (x > 0)
res = x
elsewhere
res = alpha * (exp(x) - 1)
end where
end function elu

pure function elu_prime(x, alpha) result(res)
! First derivative of the Exponential Linear Unit (ELU) activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
where (x > 0)
res = 1
elsewhere
res = alpha * exp(x)
end where
end function elu_prime

pure function exponential(x, alpha) result(res)
!! Exponential activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = exp(x)
end function exponential

pure function exponential_prime(x, alpha) result(res)
!! First derivative of the exponential activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = exp(x)
end function exponential_prime

pure function gaussian(x, alpha) result(res)
! Gaussian activation function.
real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -122,6 +165,22 @@ pure function sigmoid_prime(x, alpha) result(res)
res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
end function sigmoid_prime

pure function softplus(x, alpha) result(res)
!! Softplus activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = log(exp(x) + 1)
end function softplus

pure function softplus_prime(x, alpha) result(res)
! First derivative of the Softplus activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = 1 / (1 + exp(-x))
end function softplus_prime

pure function step(x, alpha) result(res)
! Step activation function.
real(rk), intent(in) :: x(:)
Expand Down
9 changes: 9 additions & 0 deletions src/lib/mod_dense_layer.F90
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,12 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

! assign activation function
select case(trim(activation))
case('elu')
layer % activation => elu
layer % activation_prime => elu_prime
case('exponential')
layer % activation => exponential
layer % activation_prime => exponential_prime
case('gaussian')
layer % activation => gaussian
layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +77,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
case('sigmoid')
layer % activation => sigmoid
layer % activation_prime => sigmoid_prime
case('softplus')
layer % activation => softplus
layer % activation_prime => softplus_prime
case('step')
layer % activation => step
layer % activation_prime => step_prime
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
added exponential, elu and softplus activation function#18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

mnijh
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


added exponential, elu and softplus activation function
#18
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 15 additions & 6 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -242,12 +242,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand Down

59 changes: 59 additions & 0 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,9 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +31,46 @@ end function activation_function

contains

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -122,6 +165,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

9 changes: 9 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,12 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +77,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
added exponential, elu and softplus activation function#18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

mnijh
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


added exponential, elu and softplus activation function
#18
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 15 additions & 6 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -242,12 +242,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand Down

59 changes: 59 additions & 0 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,9 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +31,46 @@ end function activation_function

contains

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -122,6 +165,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

9 changes: 9 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,12 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +77,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
added exponential, elu and softplus activation function#18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

mnijh
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


added exponential, elu and softplus activation function
#18
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 15 additions & 6 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -242,12 +242,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand Down

59 changes: 59 additions & 0 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,9 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +31,46 @@ end function activation_function

contains

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -122,6 +165,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

9 changes: 9 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,12 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +77,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
added exponential, elu and softplus activation function#18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

mnijh
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


added exponential, elu and softplus activation function
#18
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 15 additions & 6 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -242,12 +242,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand Down

59 changes: 59 additions & 0 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,9 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +31,46 @@ end function activation_function

contains

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -122,6 +165,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

9 changes: 9 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,12 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +77,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
added exponential, elu and softplus activation function#18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

mnijh
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


added exponential, elu and softplus activation function
#18
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 15 additions & 6 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -242,12 +242,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand Down

59 changes: 59 additions & 0 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,9 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +31,46 @@ end function activation_function

contains

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -122,6 +165,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

9 changes: 9 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,12 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +77,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
added exponential, elu and softplus activation function#18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

mnijh
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


added exponential, elu and softplus activation function
#18
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 15 additions & 6 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -242,12 +242,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand Down

59 changes: 59 additions & 0 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,9 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +31,46 @@ end function activation_function

contains

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -122,6 +165,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

9 changes: 9 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,12 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +77,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
added exponential, elu and softplus activation function#18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

mnijh
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


added exponential, elu and softplus activation function
#18
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


21 changes: 15 additions & 6 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -242,12 +242,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand Down

59 changes: 59 additions & 0 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,9 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +31,46 @@ end function activation_function

contains

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -122,6 +165,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

9 changes: 9 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,12 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +77,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down