Enabling double precision real number#22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sungdukyu wants to merge 9 commits into
base:master
Choose a base branch
from
Open

Enabling double precision real number #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
833a57d
added exponential, elu and softplus activation function
mnijh May 10, 2021
c50cc0b
debugged for enabling REAL64, REAL128, and INT64
sungdukyu Jun 21, 2022
fc7baf2
revert build_steps.sh to the original version
sungdukyu Jun 21, 2022
5dcc039
Merge branch 'add_activation_functions'
sungdukyu Sep 14, 2023
ad8735c
Minor modifications in convert_weights.py
sungdukyu Sep 21, 2023
5ae5266
Double precision for ClimSim E3SM implementation.
sungdukyu Sep 22, 2023
68805b6
Incorporate @singhbalwinder's bug fix on sigmoid function to avoid fl…
sungdukyu Feb 9, 2024
afac13f
Update README.md
sungdukyu Feb 21, 2024
5944cd0
Add swish (silu) activatin function.
sungdukyu Feb 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 37 additions & 12 deletions KerasWeightsProcessing/convert_weights.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['swish', 'elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -70,8 +70,9 @@ def txt_to_h5(weights_file_name, output_file_name=''):
elif layer_type == 'batchnormalization':
batchnorm_count += 4
x = BatchNormalization(name='batch_normalization_{}'.format(batchnorm_count // 4))(x)
elif layer_type == 'linear':
x = Activation('linear')(x)

elif layer_type == 'swish':
x = Activation('swish')(x)

elif not layer_type.isalpha():
if lr == False:
Expand DownExpand Up@@ -156,7 +157,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
keras_version = weights_file.attrs['keras_version']

if 'training_config' in weights_file.attrs:
training_config = weights_file.attrs['training_config'].decode('utf-8')
try:
training_config = weights_file.attrs['training_config'].decode('utf-8')
except AttributeError:
training_config = weights_file.attrs['training_config']
training_config = training_config.replace('true','True')
training_config = training_config.replace('false','False')
training_config = training_config.replace('null','None')
Expand All@@ -169,7 +173,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
learning_rate = 0.001

# Decode using the utf-8 encoding; change values for eval
model_config = weights_file.attrs['model_config'].decode('utf-8')
try:
model_config = weights_file.attrs['model_config'].decode('utf-8')
except AttributeError:
model_config = weights_file.attrs['model_config']
model_config = model_config.replace('true','True')
model_config = model_config.replace('false','False')
model_config = model_config.replace('null','None')
Expand DownExpand Up@@ -197,7 +204,7 @@ def h5_to_txt(weights_file_name, output_file_name=''):
input_layers = model_config['config'].get('input_layers',[])
input_names = [layer[0] for layer in input_layers]

else:
else: # 'Sequential'
layer_config = model_config['config']['layers']

for idx,layer in enumerate(layer_config):
Expand DownExpand Up@@ -242,12 +249,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
)
)
# add information about the activation
layer_info.append(
info_str.format(
name = activation,
info = 0
if (activation == 'elu'):
layer_info.append(
info_str.format(
name = activation,
info = 1
)
)
)
else:
layer_info.append(
info_str.format(
name = activation,
info = 0
)
)

elif class_name == 'batchnormalization':
# get beta, gamma, moving_mean, moving_variance from dictionary
for key in sorted(model_weights[name][name].keys()):
Expand DownExpand Up@@ -276,9 +292,18 @@ def h5_to_txt(weights_file_name, output_file_name=''):

elif class_name in ACTIVATIONS:
# replace previous dense layer with the advanced activation function (LeakyReLU)

try:
_tmp = layer['config']['alpha']
except KeyError:
if class_name == 'relu':
_tmp = 0 # alpha is currently not used in mod_activation
else:
raise KeyError("alpha")

layer_info[-1] = info_str.format(
name = class_name,
info = layer['config']['alpha']
info = _tmp
)

# if there are multiple outputs, remove what was just added
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -51,7 +51,7 @@ Check out an example in the [getting started notebook](https://github.com/scient
Get the code:

```
git clone https://github.com/scientific-computing/FKB
git clone https://github.com/sungdukyu/FKB64
```

Dependencies:
Expand Down
7 changes: 6 additions & 1 deletion build_steps.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,12 @@ rm -rf build
mkdir build
cd build

FC=gfortran cmake .. -DSERIAL=1
# perlmutter
module load PrgEnv-gnu/8.3.3
module load gcc/11.2.0

# FC=gfortran cmake .. -DSERIAL=1
FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90)
# FC='mpif90 -qopenmp' cmake .. -DSERIAL=1

make
Expand Down
91 changes: 88 additions & 3 deletions src/lib/mod_activation.F90
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,10 @@ module mod_activation
public :: tanhf, tanh_prime
public :: linear, linear_prime
public :: leaky_relu, leaky_relu_prime
public :: elu, elu_prime
public :: exponential, exponential_prime
public :: softplus, softplus_prime
public :: swish, swish_prime

interface
pure function activation_function(x, alpha)
Expand All@@ -28,6 +32,64 @@ end function activation_function

contains

pure function swish(x, alpha) result(res)
!! swish (or silu) activation function.
!! beta is fixed (i.e., beta==1.).
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = x * sigmoid(x, -999._rk)
end function swish

pure function swish_prime(x, alpha) result(res)
! First derivative of swish activation function.
! beta is fixed (i.e., beta==1.).
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = sigmoid(x, -999._rk) + x * sigmoid_prime(x, -999._rk)
end function swish_prime

pure function elu(x, alpha) result(res)
!! Exponential Linear Unit (ELU) activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
where (x > 0)
res = x
elsewhere
res = alpha * (exp(x) - 1)
end where
end function elu

pure function elu_prime(x, alpha) result(res)
! First derivative of the Exponential Linear Unit (ELU) activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
where (x > 0)
res = 1
elsewhere
res = alpha * exp(x)
end where
end function elu_prime

pure function exponential(x, alpha) result(res)
!! Exponential activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = exp(x)
end function exponential

pure function exponential_prime(x, alpha) result(res)
!! First derivative of the exponential activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = exp(x)
end function exponential_prime

pure function gaussian(x, alpha) result(res)
! Gaussian activation function.
real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -63,7 +125,7 @@ pure function leaky_relu_prime(x, alpha) result(res)
where (0.3 * x > 0)
res = 1
elsewhere
res = 0
res = alpha
end where
end function leaky_relu_prime

Expand DownExpand Up@@ -103,12 +165,19 @@ pure function linear_prime(x, alpha) result(res)
res = 1
end function linear_prime

! Balwinder's version (July 5th, 2022)
! - addressing floating point overflow
pure function sigmoid(x, alpha) result(res)
! Sigmoid activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = 1 / (1 + exp(-x))
real(rk) :: res(size(x)),y(size(x))
real(rk), parameter :: exp_lim = log(tiny(x)) !precision based limiter
y(:) = x(:)
where(y < exp_lim)
y = exp_lim
end where
res = 1. / (1. + exp(-y))
endfunction sigmoid

pure function sigmoid_prime(x, alpha) result(res)
Expand All@@ -122,6 +191,22 @@ pure function sigmoid_prime(x, alpha) result(res)
res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
end function sigmoid_prime

pure function softplus(x, alpha) result(res)
!! Softplus activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = log(exp(x) + 1)
end function softplus

pure function softplus_prime(x, alpha) result(res)
! First derivative of the Softplus activation function.
real(rk), intent(in) :: x(:)
real(rk), intent(in) :: alpha
real(rk) :: res(size(x))
res = 1 / (1 + exp(-x))
end function softplus_prime

pure function step(x, alpha) result(res)
! Step activation function.
real(rk), intent(in) :: x(:)
Expand Down
15 changes: 15 additions & 0 deletions src/lib/mod_dense_layer.F90
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,18 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

! assign activation function
select case(trim(activation))
case('swish')
layer % activation => swish
layer % activation_prime => swish_prime
case('silu')
layer % activation => swish
layer % activation_prime => swish_prime
case('elu')
layer % activation => elu
layer % activation_prime => elu_prime
case('exponential')
layer % activation => exponential
layer % activation_prime => exponential_prime
case('gaussian')
layer % activation => gaussian
layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +83,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
case('sigmoid')
layer % activation => sigmoid
layer % activation_prime => sigmoid_prime
case('softplus')
layer % activation => softplus
layer % activation_prime => softplus_prime
case('step')
layer % activation => step
layer % activation_prime => step_prime
Expand Down
4 changes: 4 additions & 0 deletions src/lib/mod_kinds.F90
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@ module mod_kinds
integer,parameter :: rk = real64
#elif REAL128
integer,parameter :: rk = real128
#elif REAL32
integer,parameter :: rk = real32
#else
integer,parameter :: rk = real32
#endif
Expand All@@ -19,6 +21,8 @@ module mod_kinds

#ifdef INT64
integer, parameter :: ik = int64
#elif INT32
integer, parameter :: ik = int32
#else
integer, parameter :: ik = int32
#endif
Expand Down
6 changes: 3 additions & 3 deletions src/lib/mod_layer.F90
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -68,15 +68,15 @@ end subroutine layer_backward

pure type(array1d) function array1d_constructor(length) result(a)
! Overloads the default type constructor.
integer, intent(in) :: length
integer(ik), intent(in) :: length
allocate(a % array(length))
a % array = 0
end function array1d_constructor


pure type(array2d) function array2d_constructor(dims) result(a)
! Overloads the default type constructor.
integer, intent(in) :: dims(2)
integer(ik), intent(in) :: dims(2)
allocate(a % array(dims(1), dims(2)))
a % array = 0
end function array2d_constructor
Expand DownExpand Up@@ -106,7 +106,7 @@ pure subroutine dw_init(dw, dims)
do n = 1, nm - 1
dw(n) = array2d(dims(n:n+1))
end do
dw(n) = array2d([dims(n), 1])
dw(n) = array2d([dims(n), 1_ik])
end subroutine dw_init


Expand Down
2 changes: 1 addition & 1 deletion src/lib/mod_network.F90
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -56,7 +56,7 @@ type(network_type) function net_constructor(layer_names, layer_info) result(net)

call net % init(layer_names, layer_info)

call net % sync(1)
call net % sync(1_ik)

end function net_constructor

Expand Down
2 changes: 1 addition & 1 deletion src/tests/test_ensembles.F90
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ program test_ensembles
call get_command_argument(1,args(1))

! build ensemble from members in specified directory
ensemble = ensemble_type(args(1), 0.0)
ensemble = ensemble_type(args(1), 0.0_rk)

input = [1, 2, 3, 4, 5]

Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Enabling double precision real number#22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

sungdukyu
wants to merge
9
commits into

base:master
Choose a base branch

from

 Open


Enabling double precision real number
#22
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
833a57d

added exponential, elu and softplus activation function


mnijh May 10, 2021
c50cc0b

debugged for enabling REAL64, REAL128, and INT64


sungdukyu Jun 21, 2022
fc7baf2

revert build_steps.sh to the original version


sungdukyu Jun 21, 2022
5dcc039

Merge branch 'add_activation_functions'


sungdukyu Sep 14, 2023
ad8735c

Minor modifications in convert_weights.py


sungdukyu Sep 21, 2023
5ae5266

Double precision for ClimSim E3SM implementation.


sungdukyu Sep 22, 2023
68805b6

Incorporate @singhbalwinder's bug fix on sigmoid function to avoid fl…


sungdukyu Feb 9, 2024
afac13f

Update README.md


sungdukyu Feb 21, 2024
5944cd0

Add swish (silu) activatin function.


sungdukyu Feb 23, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


49 changes: 37 additions & 12 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['swish', 'elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -70,8 +70,9 @@ def txt_to_h5(weights_file_name, output_file_name=''):
 elif layer_type == 'batchnormalization':
 batchnorm_count += 4
 x = BatchNormalization(name='batch_normalization_{}'.format(batchnorm_count // 4))(x)
 elif layer_type == 'linear':
 x = Activation('linear')(x)

 elif layer_type == 'swish':
 x = Activation('swish')(x)

 elif not layer_type.isalpha():
 if lr == False:
Expand DownExpand Up@@ -156,7 +157,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 keras_version = weights_file.attrs['keras_version']

 if 'training_config' in weights_file.attrs:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 try:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 except AttributeError: 
 training_config = weights_file.attrs['training_config']
 training_config = training_config.replace('true','True')
 training_config = training_config.replace('false','False')
 training_config = training_config.replace('null','None')
Expand All@@ -169,7 +173,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 learning_rate = 0.001

 # Decode using the utf-8 encoding; change values for eval
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 try:
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 except AttributeError:
 model_config = weights_file.attrs['model_config']
 model_config = model_config.replace('true','True')
 model_config = model_config.replace('false','False')
 model_config = model_config.replace('null','None')
Expand DownExpand Up@@ -197,7 +204,7 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 input_layers = model_config['config'].get('input_layers',[])
 input_names = [layer[0] for layer in input_layers]

 else:
 else: # 'Sequential'
 layer_config = model_config['config']['layers']

 for idx,layer in enumerate(layer_config):
Expand DownExpand Up@@ -242,12 +249,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand DownExpand Up@@ -276,9 +292,18 @@ def h5_to_txt(weights_file_name, output_file_name=''):

 elif class_name in ACTIVATIONS:
 # replace previous dense layer with the advanced activation function (LeakyReLU)

 try:
 _tmp = layer['config']['alpha']
 except KeyError:
 if class_name == 'relu':
 _tmp = 0 # alpha is currently not used in mod_activation
 else:
 raise KeyError("alpha")

 layer_info[-1] = info_str.format(
 name = class_name,
 info = layer['config']['alpha']
 info = _tmp
 )

 # if there are multiple outputs, remove what was just added
Expand Down

2 changes: 1 addition & 1 deletion
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,7 +51,7 @@ Check out an example in the [getting started notebook](https://github.com/scient
Get the code:

```
git clone https://github.com/scientific-computing/FKB
git clone https://github.com/sungdukyu/FKB64
```

Dependencies:
Expand Down

7 changes: 6 additions & 1 deletion
build_steps.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,12 @@ rm -rf build
mkdir build
cd build

FC=gfortran cmake .. -DSERIAL=1
# perlmutter
module load PrgEnv-gnu/8.3.3
module load gcc/11.2.0

# FC=gfortran cmake .. -DSERIAL=1
FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90)
# FC='mpif90 -qopenmp' cmake .. -DSERIAL=1

make
Expand Down

91 changes: 88 additions & 3 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,10 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime
 public :: swish, swish_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +32,64 @@ end function activation_function

contains

 pure function swish(x, alpha) result(res)
 !! swish (or silu) activation function.
 !! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = x * sigmoid(x, -999._rk)
 end function swish

 pure function swish_prime(x, alpha) result(res)
 ! First derivative of swish activation function.
 ! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = sigmoid(x, -999._rk) + x * sigmoid_prime(x, -999._rk)
 end function swish_prime

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -63,7 +125,7 @@ pure function leaky_relu_prime(x, alpha) result(res)
 where (0.3 * x > 0)
 res = 1
 elsewhere
 res = 0
 res = alpha
 end where
 end function leaky_relu_prime

Expand DownExpand Up@@ -103,12 +165,19 @@ pure function linear_prime(x, alpha) result(res)
 res = 1
 end function linear_prime

 ! Balwinder's version (July 5th, 2022)
 ! - addressing floating point overflow
 pure function sigmoid(x, alpha) result(res)
 ! Sigmoid activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 real(rk) :: res(size(x)),y(size(x))
 real(rk), parameter :: exp_lim = log(tiny(x)) !precision based limiter
 y(:) = x(:)
 where(y < exp_lim)
 y = exp_lim
 end where
 res = 1. / (1. + exp(-y))
 endfunction sigmoid

 pure function sigmoid_prime(x, alpha) result(res)
Expand All@@ -122,6 +191,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

15 changes: 15 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,18 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('swish')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('silu')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +83,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

4 changes: 4 additions & 0 deletions
src/lib/mod_kinds.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@ module mod_kinds
 integer,parameter :: rk = real64
#elif REAL128
 integer,parameter :: rk = real128
#elif REAL32
 integer,parameter :: rk = real32
#else
 integer,parameter :: rk = real32
#endif
Expand All@@ -19,6 +21,8 @@ module mod_kinds

#ifdef INT64
 integer, parameter :: ik = int64
#elif INT32
 integer, parameter :: ik = int32
#else
 integer, parameter :: ik = int32
#endif
Expand Down

6 changes: 3 additions & 3 deletions
src/lib/mod_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -68,15 +68,15 @@ end subroutine layer_backward

 pure type(array1d) function array1d_constructor(length) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: length
 integer(ik), intent(in) :: length
 allocate(a % array(length))
 a % array = 0
 end function array1d_constructor


 pure type(array2d) function array2d_constructor(dims) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: dims(2)
 integer(ik), intent(in) :: dims(2)
 allocate(a % array(dims(1), dims(2)))
 a % array = 0
 end function array2d_constructor
Expand DownExpand Up@@ -106,7 +106,7 @@ pure subroutine dw_init(dw, dims)
 do n = 1, nm - 1
 dw(n) = array2d(dims(n:n+1))
 end do
 dw(n) = array2d([dims(n), 1])
 dw(n) = array2d([dims(n), 1_ik])
 end subroutine dw_init


Expand Down

2 changes: 1 addition & 1 deletion
src/lib/mod_network.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -56,7 +56,7 @@ type(network_type) function net_constructor(layer_names, layer_info) result(net)

 call net % init(layer_names, layer_info)

 call net % sync(1)
 call net % sync(1_ik)

 end function net_constructor

Expand Down

2 changes: 1 addition & 1 deletion
src/tests/test_ensembles.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ program test_ensembles
 call get_command_argument(1,args(1))

 ! build ensemble from members in specified directory
 ensemble = ensemble_type(args(1), 0.0)
 ensemble = ensemble_type(args(1), 0.0_rk)

 input = [1, 2, 3, 4, 5]

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Enabling double precision real number#22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

sungdukyu
wants to merge
9
commits into

base:master
Choose a base branch

from

 Open


Enabling double precision real number
#22
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
833a57d

added exponential, elu and softplus activation function


mnijh May 10, 2021
c50cc0b

debugged for enabling REAL64, REAL128, and INT64


sungdukyu Jun 21, 2022
fc7baf2

revert build_steps.sh to the original version


sungdukyu Jun 21, 2022
5dcc039

Merge branch 'add_activation_functions'


sungdukyu Sep 14, 2023
ad8735c

Minor modifications in convert_weights.py


sungdukyu Sep 21, 2023
5ae5266

Double precision for ClimSim E3SM implementation.


sungdukyu Sep 22, 2023
68805b6

Incorporate @singhbalwinder's bug fix on sigmoid function to avoid fl…


sungdukyu Feb 9, 2024
afac13f

Update README.md


sungdukyu Feb 21, 2024
5944cd0

Add swish (silu) activatin function.


sungdukyu Feb 23, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


49 changes: 37 additions & 12 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['swish', 'elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -70,8 +70,9 @@ def txt_to_h5(weights_file_name, output_file_name=''):
 elif layer_type == 'batchnormalization':
 batchnorm_count += 4
 x = BatchNormalization(name='batch_normalization_{}'.format(batchnorm_count // 4))(x)
 elif layer_type == 'linear':
 x = Activation('linear')(x)

 elif layer_type == 'swish':
 x = Activation('swish')(x)

 elif not layer_type.isalpha():
 if lr == False:
Expand DownExpand Up@@ -156,7 +157,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 keras_version = weights_file.attrs['keras_version']

 if 'training_config' in weights_file.attrs:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 try:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 except AttributeError: 
 training_config = weights_file.attrs['training_config']
 training_config = training_config.replace('true','True')
 training_config = training_config.replace('false','False')
 training_config = training_config.replace('null','None')
Expand All@@ -169,7 +173,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 learning_rate = 0.001

 # Decode using the utf-8 encoding; change values for eval
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 try:
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 except AttributeError:
 model_config = weights_file.attrs['model_config']
 model_config = model_config.replace('true','True')
 model_config = model_config.replace('false','False')
 model_config = model_config.replace('null','None')
Expand DownExpand Up@@ -197,7 +204,7 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 input_layers = model_config['config'].get('input_layers',[])
 input_names = [layer[0] for layer in input_layers]

 else:
 else: # 'Sequential'
 layer_config = model_config['config']['layers']

 for idx,layer in enumerate(layer_config):
Expand DownExpand Up@@ -242,12 +249,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand DownExpand Up@@ -276,9 +292,18 @@ def h5_to_txt(weights_file_name, output_file_name=''):

 elif class_name in ACTIVATIONS:
 # replace previous dense layer with the advanced activation function (LeakyReLU)

 try:
 _tmp = layer['config']['alpha']
 except KeyError:
 if class_name == 'relu':
 _tmp = 0 # alpha is currently not used in mod_activation
 else:
 raise KeyError("alpha")

 layer_info[-1] = info_str.format(
 name = class_name,
 info = layer['config']['alpha']
 info = _tmp
 )

 # if there are multiple outputs, remove what was just added
Expand Down

2 changes: 1 addition & 1 deletion
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,7 +51,7 @@ Check out an example in the [getting started notebook](https://github.com/scient
Get the code:

```
git clone https://github.com/scientific-computing/FKB
git clone https://github.com/sungdukyu/FKB64
```

Dependencies:
Expand Down

7 changes: 6 additions & 1 deletion
build_steps.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,12 @@ rm -rf build
mkdir build
cd build

FC=gfortran cmake .. -DSERIAL=1
# perlmutter
module load PrgEnv-gnu/8.3.3
module load gcc/11.2.0

# FC=gfortran cmake .. -DSERIAL=1
FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90)
# FC='mpif90 -qopenmp' cmake .. -DSERIAL=1

make
Expand Down

91 changes: 88 additions & 3 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,10 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime
 public :: swish, swish_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +32,64 @@ end function activation_function

contains

 pure function swish(x, alpha) result(res)
 !! swish (or silu) activation function.
 !! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = x * sigmoid(x, -999._rk)
 end function swish

 pure function swish_prime(x, alpha) result(res)
 ! First derivative of swish activation function.
 ! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = sigmoid(x, -999._rk) + x * sigmoid_prime(x, -999._rk)
 end function swish_prime

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -63,7 +125,7 @@ pure function leaky_relu_prime(x, alpha) result(res)
 where (0.3 * x > 0)
 res = 1
 elsewhere
 res = 0
 res = alpha
 end where
 end function leaky_relu_prime

Expand DownExpand Up@@ -103,12 +165,19 @@ pure function linear_prime(x, alpha) result(res)
 res = 1
 end function linear_prime

 ! Balwinder's version (July 5th, 2022)
 ! - addressing floating point overflow
 pure function sigmoid(x, alpha) result(res)
 ! Sigmoid activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 real(rk) :: res(size(x)),y(size(x))
 real(rk), parameter :: exp_lim = log(tiny(x)) !precision based limiter
 y(:) = x(:)
 where(y < exp_lim)
 y = exp_lim
 end where
 res = 1. / (1. + exp(-y))
 endfunction sigmoid

 pure function sigmoid_prime(x, alpha) result(res)
Expand All@@ -122,6 +191,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

15 changes: 15 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,18 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('swish')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('silu')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +83,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

4 changes: 4 additions & 0 deletions
src/lib/mod_kinds.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@ module mod_kinds
 integer,parameter :: rk = real64
#elif REAL128
 integer,parameter :: rk = real128
#elif REAL32
 integer,parameter :: rk = real32
#else
 integer,parameter :: rk = real32
#endif
Expand All@@ -19,6 +21,8 @@ module mod_kinds

#ifdef INT64
 integer, parameter :: ik = int64
#elif INT32
 integer, parameter :: ik = int32
#else
 integer, parameter :: ik = int32
#endif
Expand Down

6 changes: 3 additions & 3 deletions
src/lib/mod_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -68,15 +68,15 @@ end subroutine layer_backward

 pure type(array1d) function array1d_constructor(length) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: length
 integer(ik), intent(in) :: length
 allocate(a % array(length))
 a % array = 0
 end function array1d_constructor


 pure type(array2d) function array2d_constructor(dims) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: dims(2)
 integer(ik), intent(in) :: dims(2)
 allocate(a % array(dims(1), dims(2)))
 a % array = 0
 end function array2d_constructor
Expand DownExpand Up@@ -106,7 +106,7 @@ pure subroutine dw_init(dw, dims)
 do n = 1, nm - 1
 dw(n) = array2d(dims(n:n+1))
 end do
 dw(n) = array2d([dims(n), 1])
 dw(n) = array2d([dims(n), 1_ik])
 end subroutine dw_init


Expand Down

2 changes: 1 addition & 1 deletion
src/lib/mod_network.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -56,7 +56,7 @@ type(network_type) function net_constructor(layer_names, layer_info) result(net)

 call net % init(layer_names, layer_info)

 call net % sync(1)
 call net % sync(1_ik)

 end function net_constructor

Expand Down

2 changes: 1 addition & 1 deletion
src/tests/test_ensembles.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ program test_ensembles
 call get_command_argument(1,args(1))

 ! build ensemble from members in specified directory
 ensemble = ensemble_type(args(1), 0.0)
 ensemble = ensemble_type(args(1), 0.0_rk)

 input = [1, 2, 3, 4, 5]

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Enabling double precision real number#22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

sungdukyu
wants to merge
9
commits into

base:master
Choose a base branch

from

 Open


Enabling double precision real number
#22
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
833a57d

added exponential, elu and softplus activation function


mnijh May 10, 2021
c50cc0b

debugged for enabling REAL64, REAL128, and INT64


sungdukyu Jun 21, 2022
fc7baf2

revert build_steps.sh to the original version


sungdukyu Jun 21, 2022
5dcc039

Merge branch 'add_activation_functions'


sungdukyu Sep 14, 2023
ad8735c

Minor modifications in convert_weights.py


sungdukyu Sep 21, 2023
5ae5266

Double precision for ClimSim E3SM implementation.


sungdukyu Sep 22, 2023
68805b6

Incorporate @singhbalwinder's bug fix on sigmoid function to avoid fl…


sungdukyu Feb 9, 2024
afac13f

Update README.md


sungdukyu Feb 21, 2024
5944cd0

Add swish (silu) activatin function.


sungdukyu Feb 23, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


49 changes: 37 additions & 12 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['swish', 'elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -70,8 +70,9 @@ def txt_to_h5(weights_file_name, output_file_name=''):
 elif layer_type == 'batchnormalization':
 batchnorm_count += 4
 x = BatchNormalization(name='batch_normalization_{}'.format(batchnorm_count // 4))(x)
 elif layer_type == 'linear':
 x = Activation('linear')(x)

 elif layer_type == 'swish':
 x = Activation('swish')(x)

 elif not layer_type.isalpha():
 if lr == False:
Expand DownExpand Up@@ -156,7 +157,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 keras_version = weights_file.attrs['keras_version']

 if 'training_config' in weights_file.attrs:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 try:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 except AttributeError: 
 training_config = weights_file.attrs['training_config']
 training_config = training_config.replace('true','True')
 training_config = training_config.replace('false','False')
 training_config = training_config.replace('null','None')
Expand All@@ -169,7 +173,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 learning_rate = 0.001

 # Decode using the utf-8 encoding; change values for eval
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 try:
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 except AttributeError:
 model_config = weights_file.attrs['model_config']
 model_config = model_config.replace('true','True')
 model_config = model_config.replace('false','False')
 model_config = model_config.replace('null','None')
Expand DownExpand Up@@ -197,7 +204,7 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 input_layers = model_config['config'].get('input_layers',[])
 input_names = [layer[0] for layer in input_layers]

 else:
 else: # 'Sequential'
 layer_config = model_config['config']['layers']

 for idx,layer in enumerate(layer_config):
Expand DownExpand Up@@ -242,12 +249,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand DownExpand Up@@ -276,9 +292,18 @@ def h5_to_txt(weights_file_name, output_file_name=''):

 elif class_name in ACTIVATIONS:
 # replace previous dense layer with the advanced activation function (LeakyReLU)

 try:
 _tmp = layer['config']['alpha']
 except KeyError:
 if class_name == 'relu':
 _tmp = 0 # alpha is currently not used in mod_activation
 else:
 raise KeyError("alpha")

 layer_info[-1] = info_str.format(
 name = class_name,
 info = layer['config']['alpha']
 info = _tmp
 )

 # if there are multiple outputs, remove what was just added
Expand Down

2 changes: 1 addition & 1 deletion
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,7 +51,7 @@ Check out an example in the [getting started notebook](https://github.com/scient
Get the code:

```
git clone https://github.com/scientific-computing/FKB
git clone https://github.com/sungdukyu/FKB64
```

Dependencies:
Expand Down

7 changes: 6 additions & 1 deletion
build_steps.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,12 @@ rm -rf build
mkdir build
cd build

FC=gfortran cmake .. -DSERIAL=1
# perlmutter
module load PrgEnv-gnu/8.3.3
module load gcc/11.2.0

# FC=gfortran cmake .. -DSERIAL=1
FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90)
# FC='mpif90 -qopenmp' cmake .. -DSERIAL=1

make
Expand Down

91 changes: 88 additions & 3 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,10 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime
 public :: swish, swish_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +32,64 @@ end function activation_function

contains

 pure function swish(x, alpha) result(res)
 !! swish (or silu) activation function.
 !! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = x * sigmoid(x, -999._rk)
 end function swish

 pure function swish_prime(x, alpha) result(res)
 ! First derivative of swish activation function.
 ! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = sigmoid(x, -999._rk) + x * sigmoid_prime(x, -999._rk)
 end function swish_prime

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -63,7 +125,7 @@ pure function leaky_relu_prime(x, alpha) result(res)
 where (0.3 * x > 0)
 res = 1
 elsewhere
 res = 0
 res = alpha
 end where
 end function leaky_relu_prime

Expand DownExpand Up@@ -103,12 +165,19 @@ pure function linear_prime(x, alpha) result(res)
 res = 1
 end function linear_prime

 ! Balwinder's version (July 5th, 2022)
 ! - addressing floating point overflow
 pure function sigmoid(x, alpha) result(res)
 ! Sigmoid activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 real(rk) :: res(size(x)),y(size(x))
 real(rk), parameter :: exp_lim = log(tiny(x)) !precision based limiter
 y(:) = x(:)
 where(y < exp_lim)
 y = exp_lim
 end where
 res = 1. / (1. + exp(-y))
 endfunction sigmoid

 pure function sigmoid_prime(x, alpha) result(res)
Expand All@@ -122,6 +191,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

15 changes: 15 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,18 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('swish')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('silu')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +83,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

4 changes: 4 additions & 0 deletions
src/lib/mod_kinds.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@ module mod_kinds
 integer,parameter :: rk = real64
#elif REAL128
 integer,parameter :: rk = real128
#elif REAL32
 integer,parameter :: rk = real32
#else
 integer,parameter :: rk = real32
#endif
Expand All@@ -19,6 +21,8 @@ module mod_kinds

#ifdef INT64
 integer, parameter :: ik = int64
#elif INT32
 integer, parameter :: ik = int32
#else
 integer, parameter :: ik = int32
#endif
Expand Down

6 changes: 3 additions & 3 deletions
src/lib/mod_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -68,15 +68,15 @@ end subroutine layer_backward

 pure type(array1d) function array1d_constructor(length) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: length
 integer(ik), intent(in) :: length
 allocate(a % array(length))
 a % array = 0
 end function array1d_constructor


 pure type(array2d) function array2d_constructor(dims) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: dims(2)
 integer(ik), intent(in) :: dims(2)
 allocate(a % array(dims(1), dims(2)))
 a % array = 0
 end function array2d_constructor
Expand DownExpand Up@@ -106,7 +106,7 @@ pure subroutine dw_init(dw, dims)
 do n = 1, nm - 1
 dw(n) = array2d(dims(n:n+1))
 end do
 dw(n) = array2d([dims(n), 1])
 dw(n) = array2d([dims(n), 1_ik])
 end subroutine dw_init


Expand Down

2 changes: 1 addition & 1 deletion
src/lib/mod_network.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -56,7 +56,7 @@ type(network_type) function net_constructor(layer_names, layer_info) result(net)

 call net % init(layer_names, layer_info)

 call net % sync(1)
 call net % sync(1_ik)

 end function net_constructor

Expand Down

2 changes: 1 addition & 1 deletion
src/tests/test_ensembles.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ program test_ensembles
 call get_command_argument(1,args(1))

 ! build ensemble from members in specified directory
 ensemble = ensemble_type(args(1), 0.0)
 ensemble = ensemble_type(args(1), 0.0_rk)

 input = [1, 2, 3, 4, 5]

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Enabling double precision real number#22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

sungdukyu
wants to merge
9
commits into

base:master
Choose a base branch

from

 Open


Enabling double precision real number
#22
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
833a57d

added exponential, elu and softplus activation function


mnijh May 10, 2021
c50cc0b

debugged for enabling REAL64, REAL128, and INT64


sungdukyu Jun 21, 2022
fc7baf2

revert build_steps.sh to the original version


sungdukyu Jun 21, 2022
5dcc039

Merge branch 'add_activation_functions'


sungdukyu Sep 14, 2023
ad8735c

Minor modifications in convert_weights.py


sungdukyu Sep 21, 2023
5ae5266

Double precision for ClimSim E3SM implementation.


sungdukyu Sep 22, 2023
68805b6

Incorporate @singhbalwinder's bug fix on sigmoid function to avoid fl…


sungdukyu Feb 9, 2024
afac13f

Update README.md


sungdukyu Feb 21, 2024
5944cd0

Add swish (silu) activatin function.


sungdukyu Feb 23, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


49 changes: 37 additions & 12 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['swish', 'elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -70,8 +70,9 @@ def txt_to_h5(weights_file_name, output_file_name=''):
 elif layer_type == 'batchnormalization':
 batchnorm_count += 4
 x = BatchNormalization(name='batch_normalization_{}'.format(batchnorm_count // 4))(x)
 elif layer_type == 'linear':
 x = Activation('linear')(x)

 elif layer_type == 'swish':
 x = Activation('swish')(x)

 elif not layer_type.isalpha():
 if lr == False:
Expand DownExpand Up@@ -156,7 +157,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 keras_version = weights_file.attrs['keras_version']

 if 'training_config' in weights_file.attrs:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 try:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 except AttributeError: 
 training_config = weights_file.attrs['training_config']
 training_config = training_config.replace('true','True')
 training_config = training_config.replace('false','False')
 training_config = training_config.replace('null','None')
Expand All@@ -169,7 +173,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 learning_rate = 0.001

 # Decode using the utf-8 encoding; change values for eval
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 try:
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 except AttributeError:
 model_config = weights_file.attrs['model_config']
 model_config = model_config.replace('true','True')
 model_config = model_config.replace('false','False')
 model_config = model_config.replace('null','None')
Expand DownExpand Up@@ -197,7 +204,7 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 input_layers = model_config['config'].get('input_layers',[])
 input_names = [layer[0] for layer in input_layers]

 else:
 else: # 'Sequential'
 layer_config = model_config['config']['layers']

 for idx,layer in enumerate(layer_config):
Expand DownExpand Up@@ -242,12 +249,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand DownExpand Up@@ -276,9 +292,18 @@ def h5_to_txt(weights_file_name, output_file_name=''):

 elif class_name in ACTIVATIONS:
 # replace previous dense layer with the advanced activation function (LeakyReLU)

 try:
 _tmp = layer['config']['alpha']
 except KeyError:
 if class_name == 'relu':
 _tmp = 0 # alpha is currently not used in mod_activation
 else:
 raise KeyError("alpha")

 layer_info[-1] = info_str.format(
 name = class_name,
 info = layer['config']['alpha']
 info = _tmp
 )

 # if there are multiple outputs, remove what was just added
Expand Down

2 changes: 1 addition & 1 deletion
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,7 +51,7 @@ Check out an example in the [getting started notebook](https://github.com/scient
Get the code:

```
git clone https://github.com/scientific-computing/FKB
git clone https://github.com/sungdukyu/FKB64
```

Dependencies:
Expand Down

7 changes: 6 additions & 1 deletion
build_steps.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,12 @@ rm -rf build
mkdir build
cd build

FC=gfortran cmake .. -DSERIAL=1
# perlmutter
module load PrgEnv-gnu/8.3.3
module load gcc/11.2.0

# FC=gfortran cmake .. -DSERIAL=1
FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90)
# FC='mpif90 -qopenmp' cmake .. -DSERIAL=1

make
Expand Down

91 changes: 88 additions & 3 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,10 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime
 public :: swish, swish_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +32,64 @@ end function activation_function

contains

 pure function swish(x, alpha) result(res)
 !! swish (or silu) activation function.
 !! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = x * sigmoid(x, -999._rk)
 end function swish

 pure function swish_prime(x, alpha) result(res)
 ! First derivative of swish activation function.
 ! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = sigmoid(x, -999._rk) + x * sigmoid_prime(x, -999._rk)
 end function swish_prime

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -63,7 +125,7 @@ pure function leaky_relu_prime(x, alpha) result(res)
 where (0.3 * x > 0)
 res = 1
 elsewhere
 res = 0
 res = alpha
 end where
 end function leaky_relu_prime

Expand DownExpand Up@@ -103,12 +165,19 @@ pure function linear_prime(x, alpha) result(res)
 res = 1
 end function linear_prime

 ! Balwinder's version (July 5th, 2022)
 ! - addressing floating point overflow
 pure function sigmoid(x, alpha) result(res)
 ! Sigmoid activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 real(rk) :: res(size(x)),y(size(x))
 real(rk), parameter :: exp_lim = log(tiny(x)) !precision based limiter
 y(:) = x(:)
 where(y < exp_lim)
 y = exp_lim
 end where
 res = 1. / (1. + exp(-y))
 endfunction sigmoid

 pure function sigmoid_prime(x, alpha) result(res)
Expand All@@ -122,6 +191,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

15 changes: 15 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,18 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('swish')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('silu')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +83,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

4 changes: 4 additions & 0 deletions
src/lib/mod_kinds.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@ module mod_kinds
 integer,parameter :: rk = real64
#elif REAL128
 integer,parameter :: rk = real128
#elif REAL32
 integer,parameter :: rk = real32
#else
 integer,parameter :: rk = real32
#endif
Expand All@@ -19,6 +21,8 @@ module mod_kinds

#ifdef INT64
 integer, parameter :: ik = int64
#elif INT32
 integer, parameter :: ik = int32
#else
 integer, parameter :: ik = int32
#endif
Expand Down

6 changes: 3 additions & 3 deletions
src/lib/mod_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -68,15 +68,15 @@ end subroutine layer_backward

 pure type(array1d) function array1d_constructor(length) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: length
 integer(ik), intent(in) :: length
 allocate(a % array(length))
 a % array = 0
 end function array1d_constructor


 pure type(array2d) function array2d_constructor(dims) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: dims(2)
 integer(ik), intent(in) :: dims(2)
 allocate(a % array(dims(1), dims(2)))
 a % array = 0
 end function array2d_constructor
Expand DownExpand Up@@ -106,7 +106,7 @@ pure subroutine dw_init(dw, dims)
 do n = 1, nm - 1
 dw(n) = array2d(dims(n:n+1))
 end do
 dw(n) = array2d([dims(n), 1])
 dw(n) = array2d([dims(n), 1_ik])
 end subroutine dw_init


Expand Down

2 changes: 1 addition & 1 deletion
src/lib/mod_network.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -56,7 +56,7 @@ type(network_type) function net_constructor(layer_names, layer_info) result(net)

 call net % init(layer_names, layer_info)

 call net % sync(1)
 call net % sync(1_ik)

 end function net_constructor

Expand Down

2 changes: 1 addition & 1 deletion
src/tests/test_ensembles.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ program test_ensembles
 call get_command_argument(1,args(1))

 ! build ensemble from members in specified directory
 ensemble = ensemble_type(args(1), 0.0)
 ensemble = ensemble_type(args(1), 0.0_rk)

 input = [1, 2, 3, 4, 5]

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Enabling double precision real number#22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

sungdukyu
wants to merge
9
commits into

base:master
Choose a base branch

from

 Open


Enabling double precision real number
#22
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
833a57d

added exponential, elu and softplus activation function


mnijh May 10, 2021
c50cc0b

debugged for enabling REAL64, REAL128, and INT64


sungdukyu Jun 21, 2022
fc7baf2

revert build_steps.sh to the original version


sungdukyu Jun 21, 2022
5dcc039

Merge branch 'add_activation_functions'


sungdukyu Sep 14, 2023
ad8735c

Minor modifications in convert_weights.py


sungdukyu Sep 21, 2023
5ae5266

Double precision for ClimSim E3SM implementation.


sungdukyu Sep 22, 2023
68805b6

Incorporate @singhbalwinder's bug fix on sigmoid function to avoid fl…


sungdukyu Feb 9, 2024
afac13f

Update README.md


sungdukyu Feb 21, 2024
5944cd0

Add swish (silu) activatin function.


sungdukyu Feb 23, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


49 changes: 37 additions & 12 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['swish', 'elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -70,8 +70,9 @@ def txt_to_h5(weights_file_name, output_file_name=''):
 elif layer_type == 'batchnormalization':
 batchnorm_count += 4
 x = BatchNormalization(name='batch_normalization_{}'.format(batchnorm_count // 4))(x)
 elif layer_type == 'linear':
 x = Activation('linear')(x)

 elif layer_type == 'swish':
 x = Activation('swish')(x)

 elif not layer_type.isalpha():
 if lr == False:
Expand DownExpand Up@@ -156,7 +157,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 keras_version = weights_file.attrs['keras_version']

 if 'training_config' in weights_file.attrs:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 try:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 except AttributeError: 
 training_config = weights_file.attrs['training_config']
 training_config = training_config.replace('true','True')
 training_config = training_config.replace('false','False')
 training_config = training_config.replace('null','None')
Expand All@@ -169,7 +173,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 learning_rate = 0.001

 # Decode using the utf-8 encoding; change values for eval
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 try:
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 except AttributeError:
 model_config = weights_file.attrs['model_config']
 model_config = model_config.replace('true','True')
 model_config = model_config.replace('false','False')
 model_config = model_config.replace('null','None')
Expand DownExpand Up@@ -197,7 +204,7 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 input_layers = model_config['config'].get('input_layers',[])
 input_names = [layer[0] for layer in input_layers]

 else:
 else: # 'Sequential'
 layer_config = model_config['config']['layers']

 for idx,layer in enumerate(layer_config):
Expand DownExpand Up@@ -242,12 +249,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand DownExpand Up@@ -276,9 +292,18 @@ def h5_to_txt(weights_file_name, output_file_name=''):

 elif class_name in ACTIVATIONS:
 # replace previous dense layer with the advanced activation function (LeakyReLU)

 try:
 _tmp = layer['config']['alpha']
 except KeyError:
 if class_name == 'relu':
 _tmp = 0 # alpha is currently not used in mod_activation
 else:
 raise KeyError("alpha")

 layer_info[-1] = info_str.format(
 name = class_name,
 info = layer['config']['alpha']
 info = _tmp
 )

 # if there are multiple outputs, remove what was just added
Expand Down

2 changes: 1 addition & 1 deletion
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,7 +51,7 @@ Check out an example in the [getting started notebook](https://github.com/scient
Get the code:

```
git clone https://github.com/scientific-computing/FKB
git clone https://github.com/sungdukyu/FKB64
```

Dependencies:
Expand Down

7 changes: 6 additions & 1 deletion
build_steps.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,12 @@ rm -rf build
mkdir build
cd build

FC=gfortran cmake .. -DSERIAL=1
# perlmutter
module load PrgEnv-gnu/8.3.3
module load gcc/11.2.0

# FC=gfortran cmake .. -DSERIAL=1
FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90)
# FC='mpif90 -qopenmp' cmake .. -DSERIAL=1

make
Expand Down

91 changes: 88 additions & 3 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,10 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime
 public :: swish, swish_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +32,64 @@ end function activation_function

contains

 pure function swish(x, alpha) result(res)
 !! swish (or silu) activation function.
 !! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = x * sigmoid(x, -999._rk)
 end function swish

 pure function swish_prime(x, alpha) result(res)
 ! First derivative of swish activation function.
 ! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = sigmoid(x, -999._rk) + x * sigmoid_prime(x, -999._rk)
 end function swish_prime

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -63,7 +125,7 @@ pure function leaky_relu_prime(x, alpha) result(res)
 where (0.3 * x > 0)
 res = 1
 elsewhere
 res = 0
 res = alpha
 end where
 end function leaky_relu_prime

Expand DownExpand Up@@ -103,12 +165,19 @@ pure function linear_prime(x, alpha) result(res)
 res = 1
 end function linear_prime

 ! Balwinder's version (July 5th, 2022)
 ! - addressing floating point overflow
 pure function sigmoid(x, alpha) result(res)
 ! Sigmoid activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 real(rk) :: res(size(x)),y(size(x))
 real(rk), parameter :: exp_lim = log(tiny(x)) !precision based limiter
 y(:) = x(:)
 where(y < exp_lim)
 y = exp_lim
 end where
 res = 1. / (1. + exp(-y))
 endfunction sigmoid

 pure function sigmoid_prime(x, alpha) result(res)
Expand All@@ -122,6 +191,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

15 changes: 15 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,18 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('swish')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('silu')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +83,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

4 changes: 4 additions & 0 deletions
src/lib/mod_kinds.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@ module mod_kinds
 integer,parameter :: rk = real64
#elif REAL128
 integer,parameter :: rk = real128
#elif REAL32
 integer,parameter :: rk = real32
#else
 integer,parameter :: rk = real32
#endif
Expand All@@ -19,6 +21,8 @@ module mod_kinds

#ifdef INT64
 integer, parameter :: ik = int64
#elif INT32
 integer, parameter :: ik = int32
#else
 integer, parameter :: ik = int32
#endif
Expand Down

6 changes: 3 additions & 3 deletions
src/lib/mod_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -68,15 +68,15 @@ end subroutine layer_backward

 pure type(array1d) function array1d_constructor(length) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: length
 integer(ik), intent(in) :: length
 allocate(a % array(length))
 a % array = 0
 end function array1d_constructor


 pure type(array2d) function array2d_constructor(dims) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: dims(2)
 integer(ik), intent(in) :: dims(2)
 allocate(a % array(dims(1), dims(2)))
 a % array = 0
 end function array2d_constructor
Expand DownExpand Up@@ -106,7 +106,7 @@ pure subroutine dw_init(dw, dims)
 do n = 1, nm - 1
 dw(n) = array2d(dims(n:n+1))
 end do
 dw(n) = array2d([dims(n), 1])
 dw(n) = array2d([dims(n), 1_ik])
 end subroutine dw_init


Expand Down

2 changes: 1 addition & 1 deletion
src/lib/mod_network.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -56,7 +56,7 @@ type(network_type) function net_constructor(layer_names, layer_info) result(net)

 call net % init(layer_names, layer_info)

 call net % sync(1)
 call net % sync(1_ik)

 end function net_constructor

Expand Down

2 changes: 1 addition & 1 deletion
src/tests/test_ensembles.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ program test_ensembles
 call get_command_argument(1,args(1))

 ! build ensemble from members in specified directory
 ensemble = ensemble_type(args(1), 0.0)
 ensemble = ensemble_type(args(1), 0.0_rk)

 input = [1, 2, 3, 4, 5]

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Enabling double precision real number#22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

sungdukyu
wants to merge
9
commits into

base:master
Choose a base branch

from

 Open


Enabling double precision real number
#22
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
833a57d

added exponential, elu and softplus activation function


mnijh May 10, 2021
c50cc0b

debugged for enabling REAL64, REAL128, and INT64


sungdukyu Jun 21, 2022
fc7baf2

revert build_steps.sh to the original version


sungdukyu Jun 21, 2022
5dcc039

Merge branch 'add_activation_functions'


sungdukyu Sep 14, 2023
ad8735c

Minor modifications in convert_weights.py


sungdukyu Sep 21, 2023
5ae5266

Double precision for ClimSim E3SM implementation.


sungdukyu Sep 22, 2023
68805b6

Incorporate @singhbalwinder's bug fix on sigmoid function to avoid fl…


sungdukyu Feb 9, 2024
afac13f

Update README.md


sungdukyu Feb 21, 2024
5944cd0

Add swish (silu) activatin function.


sungdukyu Feb 23, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


49 changes: 37 additions & 12 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['swish', 'elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -70,8 +70,9 @@ def txt_to_h5(weights_file_name, output_file_name=''):
 elif layer_type == 'batchnormalization':
 batchnorm_count += 4
 x = BatchNormalization(name='batch_normalization_{}'.format(batchnorm_count // 4))(x)
 elif layer_type == 'linear':
 x = Activation('linear')(x)

 elif layer_type == 'swish':
 x = Activation('swish')(x)

 elif not layer_type.isalpha():
 if lr == False:
Expand DownExpand Up@@ -156,7 +157,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 keras_version = weights_file.attrs['keras_version']

 if 'training_config' in weights_file.attrs:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 try:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 except AttributeError: 
 training_config = weights_file.attrs['training_config']
 training_config = training_config.replace('true','True')
 training_config = training_config.replace('false','False')
 training_config = training_config.replace('null','None')
Expand All@@ -169,7 +173,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 learning_rate = 0.001

 # Decode using the utf-8 encoding; change values for eval
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 try:
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 except AttributeError:
 model_config = weights_file.attrs['model_config']
 model_config = model_config.replace('true','True')
 model_config = model_config.replace('false','False')
 model_config = model_config.replace('null','None')
Expand DownExpand Up@@ -197,7 +204,7 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 input_layers = model_config['config'].get('input_layers',[])
 input_names = [layer[0] for layer in input_layers]

 else:
 else: # 'Sequential'
 layer_config = model_config['config']['layers']

 for idx,layer in enumerate(layer_config):
Expand DownExpand Up@@ -242,12 +249,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand DownExpand Up@@ -276,9 +292,18 @@ def h5_to_txt(weights_file_name, output_file_name=''):

 elif class_name in ACTIVATIONS:
 # replace previous dense layer with the advanced activation function (LeakyReLU)

 try:
 _tmp = layer['config']['alpha']
 except KeyError:
 if class_name == 'relu':
 _tmp = 0 # alpha is currently not used in mod_activation
 else:
 raise KeyError("alpha")

 layer_info[-1] = info_str.format(
 name = class_name,
 info = layer['config']['alpha']
 info = _tmp
 )

 # if there are multiple outputs, remove what was just added
Expand Down

2 changes: 1 addition & 1 deletion
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,7 +51,7 @@ Check out an example in the [getting started notebook](https://github.com/scient
Get the code:

```
git clone https://github.com/scientific-computing/FKB
git clone https://github.com/sungdukyu/FKB64
```

Dependencies:
Expand Down

7 changes: 6 additions & 1 deletion
build_steps.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,12 @@ rm -rf build
mkdir build
cd build

FC=gfortran cmake .. -DSERIAL=1
# perlmutter
module load PrgEnv-gnu/8.3.3
module load gcc/11.2.0

# FC=gfortran cmake .. -DSERIAL=1
FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90)
# FC='mpif90 -qopenmp' cmake .. -DSERIAL=1

make
Expand Down

91 changes: 88 additions & 3 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,10 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime
 public :: swish, swish_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +32,64 @@ end function activation_function

contains

 pure function swish(x, alpha) result(res)
 !! swish (or silu) activation function.
 !! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = x * sigmoid(x, -999._rk)
 end function swish

 pure function swish_prime(x, alpha) result(res)
 ! First derivative of swish activation function.
 ! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = sigmoid(x, -999._rk) + x * sigmoid_prime(x, -999._rk)
 end function swish_prime

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -63,7 +125,7 @@ pure function leaky_relu_prime(x, alpha) result(res)
 where (0.3 * x > 0)
 res = 1
 elsewhere
 res = 0
 res = alpha
 end where
 end function leaky_relu_prime

Expand DownExpand Up@@ -103,12 +165,19 @@ pure function linear_prime(x, alpha) result(res)
 res = 1
 end function linear_prime

 ! Balwinder's version (July 5th, 2022)
 ! - addressing floating point overflow
 pure function sigmoid(x, alpha) result(res)
 ! Sigmoid activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 real(rk) :: res(size(x)),y(size(x))
 real(rk), parameter :: exp_lim = log(tiny(x)) !precision based limiter
 y(:) = x(:)
 where(y < exp_lim)
 y = exp_lim
 end where
 res = 1. / (1. + exp(-y))
 endfunction sigmoid

 pure function sigmoid_prime(x, alpha) result(res)
Expand All@@ -122,6 +191,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

15 changes: 15 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,18 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('swish')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('silu')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +83,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

4 changes: 4 additions & 0 deletions
src/lib/mod_kinds.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@ module mod_kinds
 integer,parameter :: rk = real64
#elif REAL128
 integer,parameter :: rk = real128
#elif REAL32
 integer,parameter :: rk = real32
#else
 integer,parameter :: rk = real32
#endif
Expand All@@ -19,6 +21,8 @@ module mod_kinds

#ifdef INT64
 integer, parameter :: ik = int64
#elif INT32
 integer, parameter :: ik = int32
#else
 integer, parameter :: ik = int32
#endif
Expand Down

6 changes: 3 additions & 3 deletions
src/lib/mod_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -68,15 +68,15 @@ end subroutine layer_backward

 pure type(array1d) function array1d_constructor(length) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: length
 integer(ik), intent(in) :: length
 allocate(a % array(length))
 a % array = 0
 end function array1d_constructor


 pure type(array2d) function array2d_constructor(dims) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: dims(2)
 integer(ik), intent(in) :: dims(2)
 allocate(a % array(dims(1), dims(2)))
 a % array = 0
 end function array2d_constructor
Expand DownExpand Up@@ -106,7 +106,7 @@ pure subroutine dw_init(dw, dims)
 do n = 1, nm - 1
 dw(n) = array2d(dims(n:n+1))
 end do
 dw(n) = array2d([dims(n), 1])
 dw(n) = array2d([dims(n), 1_ik])
 end subroutine dw_init


Expand Down

2 changes: 1 addition & 1 deletion
src/lib/mod_network.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -56,7 +56,7 @@ type(network_type) function net_constructor(layer_names, layer_info) result(net)

 call net % init(layer_names, layer_info)

 call net % sync(1)
 call net % sync(1_ik)

 end function net_constructor

Expand Down

2 changes: 1 addition & 1 deletion
src/tests/test_ensembles.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ program test_ensembles
 call get_command_argument(1,args(1))

 ! build ensemble from members in specified directory
 ensemble = ensemble_type(args(1), 0.0)
 ensemble = ensemble_type(args(1), 0.0_rk)

 input = [1, 2, 3, 4, 5]

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Enabling double precision real number#22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

sungdukyu
wants to merge
9
commits into

base:master
Choose a base branch

from

 Open


Enabling double precision real number
#22
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
833a57d

added exponential, elu and softplus activation function


mnijh May 10, 2021
c50cc0b

debugged for enabling REAL64, REAL128, and INT64


sungdukyu Jun 21, 2022
fc7baf2

revert build_steps.sh to the original version


sungdukyu Jun 21, 2022
5dcc039

Merge branch 'add_activation_functions'


sungdukyu Sep 14, 2023
ad8735c

Minor modifications in convert_weights.py


sungdukyu Sep 21, 2023
5ae5266

Double precision for ClimSim E3SM implementation.


sungdukyu Sep 22, 2023
68805b6

Incorporate @singhbalwinder's bug fix on sigmoid function to avoid fl…


sungdukyu Feb 9, 2024
afac13f

Update README.md


sungdukyu Feb 21, 2024
5944cd0

Add swish (silu) activatin function.


sungdukyu Feb 23, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


49 changes: 37 additions & 12 deletions
KerasWeightsProcessing/convert_weights.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
from keras import optimizers

INPUT = ['input']
ACTIVATIONS = ['relu', 'linear', 'leakyrelu', 'sigmoid']
ACTIVATIONS = ['swish', 'elu', 'exponential', 'relu', 'linear', 'leakyrelu', 'softplus', 'sigmoid', 'tanh']
SUPPORTED_LAYERS = ['dense', 'dropout', 'batchnormalization'] + ACTIVATIONS + INPUT

def txt_to_h5(weights_file_name, output_file_name=''):
Expand DownExpand Up@@ -70,8 +70,9 @@ def txt_to_h5(weights_file_name, output_file_name=''):
 elif layer_type == 'batchnormalization':
 batchnorm_count += 4
 x = BatchNormalization(name='batch_normalization_{}'.format(batchnorm_count // 4))(x)
 elif layer_type == 'linear':
 x = Activation('linear')(x)

 elif layer_type == 'swish':
 x = Activation('swish')(x)

 elif not layer_type.isalpha():
 if lr == False:
Expand DownExpand Up@@ -156,7 +157,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 keras_version = weights_file.attrs['keras_version']

 if 'training_config' in weights_file.attrs:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 try:
 training_config = weights_file.attrs['training_config'].decode('utf-8')
 except AttributeError: 
 training_config = weights_file.attrs['training_config']
 training_config = training_config.replace('true','True')
 training_config = training_config.replace('false','False')
 training_config = training_config.replace('null','None')
Expand All@@ -169,7 +173,10 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 learning_rate = 0.001

 # Decode using the utf-8 encoding; change values for eval
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 try:
 model_config = weights_file.attrs['model_config'].decode('utf-8')
 except AttributeError:
 model_config = weights_file.attrs['model_config']
 model_config = model_config.replace('true','True')
 model_config = model_config.replace('false','False')
 model_config = model_config.replace('null','None')
Expand DownExpand Up@@ -197,7 +204,7 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 input_layers = model_config['config'].get('input_layers',[])
 input_names = [layer[0] for layer in input_layers]

 else:
 else: # 'Sequential'
 layer_config = model_config['config']['layers']

 for idx,layer in enumerate(layer_config):
Expand DownExpand Up@@ -242,12 +249,21 @@ def h5_to_txt(weights_file_name, output_file_name=''):
 )
 )
 # add information about the activation
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 if (activation == 'elu'):
 layer_info.append(
 info_str.format(
 name = activation,
 info = 1
 )
 )
 )
 else:
 layer_info.append(
 info_str.format(
 name = activation,
 info = 0
 )
 )

 elif class_name == 'batchnormalization':
 # get beta, gamma, moving_mean, moving_variance from dictionary
 for key in sorted(model_weights[name][name].keys()):
Expand DownExpand Up@@ -276,9 +292,18 @@ def h5_to_txt(weights_file_name, output_file_name=''):

 elif class_name in ACTIVATIONS:
 # replace previous dense layer with the advanced activation function (LeakyReLU)

 try:
 _tmp = layer['config']['alpha']
 except KeyError:
 if class_name == 'relu':
 _tmp = 0 # alpha is currently not used in mod_activation
 else:
 raise KeyError("alpha")

 layer_info[-1] = info_str.format(
 name = class_name,
 info = layer['config']['alpha']
 info = _tmp
 )

 # if there are multiple outputs, remove what was just added
Expand Down

2 changes: 1 addition & 1 deletion
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,7 +51,7 @@ Check out an example in the [getting started notebook](https://github.com/scient
Get the code:

```
git clone https://github.com/scientific-computing/FKB
git clone https://github.com/sungdukyu/FKB64
```

Dependencies:
Expand Down

7 changes: 6 additions & 1 deletion
build_steps.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,12 @@ rm -rf build
mkdir build
cd build

FC=gfortran cmake .. -DSERIAL=1
# perlmutter
module load PrgEnv-gnu/8.3.3
module load gcc/11.2.0

# FC=gfortran cmake .. -DSERIAL=1
FC=gfortran cmake .. -DSERIAL=1 -DREAL=64 # enable double precision real number (note that the default precision for real number is single. See src/lib/mod_kinds.F90)
# FC='mpif90 -qopenmp' cmake .. -DSERIAL=1

make
Expand Down

91 changes: 88 additions & 3 deletions
src/lib/mod_activation.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,10 @@ module mod_activation
 public :: tanhf, tanh_prime
 public :: linear, linear_prime
 public :: leaky_relu, leaky_relu_prime
 public :: elu, elu_prime
 public :: exponential, exponential_prime
 public :: softplus, softplus_prime
 public :: swish, swish_prime

 interface
 pure function activation_function(x, alpha)
Expand All@@ -28,6 +32,64 @@ end function activation_function

contains

 pure function swish(x, alpha) result(res)
 !! swish (or silu) activation function.
 !! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = x * sigmoid(x, -999._rk)
 end function swish

 pure function swish_prime(x, alpha) result(res)
 ! First derivative of swish activation function.
 ! beta is fixed (i.e., beta==1.).
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = sigmoid(x, -999._rk) + x * sigmoid_prime(x, -999._rk)
 end function swish_prime

 pure function elu(x, alpha) result(res)
 !! Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = x
 elsewhere
 res = alpha * (exp(x) - 1)
 end where
 end function elu

 pure function elu_prime(x, alpha) result(res)
 ! First derivative of the Exponential Linear Unit (ELU) activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 where (x > 0)
 res = 1
 elsewhere
 res = alpha * exp(x)
 end where
 end function elu_prime

 pure function exponential(x, alpha) result(res)
 !! Exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential

 pure function exponential_prime(x, alpha) result(res)
 !! First derivative of the exponential activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = exp(x)
 end function exponential_prime

 pure function gaussian(x, alpha) result(res)
 ! Gaussian activation function.
 real(rk), intent(in) :: x(:)
Expand DownExpand Up@@ -63,7 +125,7 @@ pure function leaky_relu_prime(x, alpha) result(res)
 where (0.3 * x > 0)
 res = 1
 elsewhere
 res = 0
 res = alpha
 end where
 end function leaky_relu_prime

Expand DownExpand Up@@ -103,12 +165,19 @@ pure function linear_prime(x, alpha) result(res)
 res = 1
 end function linear_prime

 ! Balwinder's version (July 5th, 2022)
 ! - addressing floating point overflow
 pure function sigmoid(x, alpha) result(res)
 ! Sigmoid activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 real(rk) :: res(size(x)),y(size(x))
 real(rk), parameter :: exp_lim = log(tiny(x)) !precision based limiter
 y(:) = x(:)
 where(y < exp_lim)
 y = exp_lim
 end where
 res = 1. / (1. + exp(-y))
 endfunction sigmoid

 pure function sigmoid_prime(x, alpha) result(res)
Expand All@@ -122,6 +191,22 @@ pure function sigmoid_prime(x, alpha) result(res)
 res = sigmoid(x, tmp_alpha) * (1 - sigmoid(x, tmp_alpha))
 end function sigmoid_prime

 pure function softplus(x, alpha) result(res)
 !! Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = log(exp(x) + 1)
 end function softplus

 pure function softplus_prime(x, alpha) result(res)
 ! First derivative of the Softplus activation function.
 real(rk), intent(in) :: x(:)
 real(rk), intent(in) :: alpha
 real(rk) :: res(size(x))
 res = 1 / (1 + exp(-x))
 end function softplus_prime

 pure function step(x, alpha) result(res)
 ! Step activation function.
 real(rk), intent(in) :: x(:)
Expand Down

15 changes: 15 additions & 0 deletions
src/lib/mod_dense_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -59,6 +59,18 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result

 ! assign activation function
 select case(trim(activation))
 case('swish')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('silu')
 layer % activation => swish
 layer % activation_prime => swish_prime
 case('elu')
 layer % activation => elu
 layer % activation_prime => elu_prime
 case('exponential')
 layer % activation => exponential
 layer % activation_prime => exponential_prime
 case('gaussian')
 layer % activation => gaussian
 layer % activation_prime => gaussian_prime
Expand All@@ -71,6 +83,9 @@ type(Dense) function constructor(this_size, next_size, activation, alpha) result
 case('sigmoid')
 layer % activation => sigmoid
 layer % activation_prime => sigmoid_prime
 case('softplus')
 layer % activation => softplus
 layer % activation_prime => softplus_prime
 case('step')
 layer % activation => step
 layer % activation_prime => step_prime
Expand Down

4 changes: 4 additions & 0 deletions
src/lib/mod_kinds.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@ module mod_kinds
 integer,parameter :: rk = real64
#elif REAL128
 integer,parameter :: rk = real128
#elif REAL32
 integer,parameter :: rk = real32
#else
 integer,parameter :: rk = real32
#endif
Expand All@@ -19,6 +21,8 @@ module mod_kinds

#ifdef INT64
 integer, parameter :: ik = int64
#elif INT32
 integer, parameter :: ik = int32
#else
 integer, parameter :: ik = int32
#endif
Expand Down

6 changes: 3 additions & 3 deletions
src/lib/mod_layer.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -68,15 +68,15 @@ end subroutine layer_backward

 pure type(array1d) function array1d_constructor(length) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: length
 integer(ik), intent(in) :: length
 allocate(a % array(length))
 a % array = 0
 end function array1d_constructor


 pure type(array2d) function array2d_constructor(dims) result(a)
 ! Overloads the default type constructor.
 integer, intent(in) :: dims(2)
 integer(ik), intent(in) :: dims(2)
 allocate(a % array(dims(1), dims(2)))
 a % array = 0
 end function array2d_constructor
Expand DownExpand Up@@ -106,7 +106,7 @@ pure subroutine dw_init(dw, dims)
 do n = 1, nm - 1
 dw(n) = array2d(dims(n:n+1))
 end do
 dw(n) = array2d([dims(n), 1])
 dw(n) = array2d([dims(n), 1_ik])
 end subroutine dw_init


Expand Down

2 changes: 1 addition & 1 deletion
src/lib/mod_network.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -56,7 +56,7 @@ type(network_type) function net_constructor(layer_names, layer_info) result(net)

 call net % init(layer_names, layer_info)

 call net % sync(1)
 call net % sync(1_ik)

 end function net_constructor

Expand Down

2 changes: 1 addition & 1 deletion
src/tests/test_ensembles.F90

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ program test_ensembles
 call get_command_argument(1,args(1))

 ! build ensemble from members in specified directory
 ensemble = ensemble_type(args(1), 0.0)
 ensemble = ensemble_type(args(1), 0.0_rk)

 input = [1, 2, 3, 4, 5]

Expand Down