abstracts[] |
{'sha1': '1e65e04ac2d466a006864266bf3e012031743573', 'content': 'Inspired by the adaptation phenomenon of neuronal firing, we propose the\nregularity normalization (RN) as an unsupervised attention mechanism (UAM)\nwhich computes the statistical regularity in the implicit space of neural\nnetworks under the Minimum Description Length (MDL) principle. Treating the\nneural network optimization process as a partially observable model selection\nproblem, the regularity normalization constrains the implicit space by a\nnormalization factor, the universal code length. We compute this universal code\nincrementally across neural network layers and demonstrate the flexibility to\ninclude data priors such as top-down attention and other oracle information.\nEmpirically, our approach outperforms existing normalization methods in\ntackling limited, imbalanced and non-stationary input distribution in image\nclassification, classic control, procedurally-generated reinforcement learning,\ngenerative modeling, handwriting generation and question answering tasks with\nvarious neural network architectures. Lastly, the unsupervised attention\nmechanisms is a useful probing tool for neural networks by tracking the\ndependency and critical learning stages across layers and recurrent time steps\nof deep networks.', 'mimetype': 'text/plain', 'lang': 'en'}
|
container |
|
container_id |
|
contribs[] |
{'index': 0, 'creator_id': None, 'creator': None, 'raw_name': 'Baihan Lin', 'given_name': None, 'surname': None, 'role': 'author', 'raw_affiliation': None, 'extra': None}
|
ext_ids |
{'doi': None, 'wikidata_qid': None, 'isbn13': None, 'pmid': None, 'pmcid': None, 'core': None, 'arxiv': '1902.10658v13', 'jstor': None, 'ark': None, 'mag': None, 'doaj': None, 'dblp': None, 'oai': None, 'hdl': None}
|
files |
[]
|
filesets |
[]
|
issue |
|
language |
en
|
license_slug |
ARXIV-1.0
|
number |
|
original_title |
|
pages |
|
publisher |
|
refs |
[]
|
release_date |
2021-12-23
|
release_stage |
submitted
|
release_type |
article
|
release_year |
2021
|
subtitle |
|
title |
Regularity Normalization: Neuroscience-Inspired Unsupervised Attention across Neural Network Layers
|
version |
v13
|
volume |
|
webcaptures |
[]
|
withdrawn_date |
|
withdrawn_status |
|
withdrawn_year |
|
work_id |
2ulnzgzuyzelrhbls23xmvmyt4
|