abstracts[] |
{'sha1': '00933c39a01a15c3728b281dbf89de3551f5c0f0', 'content': 'Inspired by the adaptation phenomenon of neuronal firing, we propose the\nregularity normalization (RN) as an unsupervised attention mechanism (UAM)\nwhich computes the statistical regularity in the implicit space of neural\nnetworks under the Minimum Description Length (MDL) principle. Treating the\nneural network optimization process as a partially observable model selection\nproblem, UAM constrains the implicit space by a normalization factor, the\nuniversal code length. We compute this universal code incrementally across\nneural network layers and demonstrated the flexibility to include data priors\nsuch as top-down attention and other oracle information. Empirically, our\napproach outperforms existing normalization methods in tackling limited,\nimbalanced and non-stationary input distribution in image classification,\nclassic control, procedurally-generated reinforcement learning, generative\nmodeling, handwriting generation and question answering tasks with various\nneural network architectures. Lastly, UAM tracks dependency and critical\nlearning stages across layers and recurrent time steps of deep networks.', 'mimetype': 'text/plain', 'lang': 'en'}
|
container |
|
container_id |
|
contribs[] |
{'index': 0, 'creator_id': None, 'creator': None, 'raw_name': 'Baihan Lin', 'given_name': None, 'surname': None, 'role': 'author', 'raw_affiliation': None, 'extra': None}
|
ext_ids |
{'doi': None, 'wikidata_qid': None, 'isbn13': None, 'pmid': None, 'pmcid': None, 'core': None, 'arxiv': '1902.10658v11', 'jstor': None, 'ark': None, 'mag': None, 'doaj': None, 'dblp': None, 'oai': None, 'hdl': None}
|
files[] |
{'state': 'active', 'ident': 'v6x2y2j4cjgavbt7eswj2pfkci', 'revision': '76cc7ee1-1865-4d3b-9ada-9947af02dfde', 'redirect': None, 'extra': None, 'edit_extra': None, 'size': 6856602, 'md5': '918e5379e04609d8e0a0badfec8f1f7e', 'sha1': 'c129249a1f1f604d7f5ac42fba855be27c0d4237', 'sha256': '96f075eaa33b97282317e891adfedc28bae138be3b8d89eae8b702f2442f77fe', 'urls': [{'url': 'https://arxiv.org/pdf/1902.10658v11.pdf', 'rel': 'repository'}, {'url': 'https://web.archive.org/web/20200610025819/https://arxiv.org/pdf/1902.10658v11.pdf', 'rel': 'webarchive'}], 'mimetype': 'application/pdf', 'content_scope': None, 'release_ids': ['in4kcsasnzhzhds2y5ou277lh4'], 'releases': None}
|
filesets |
[]
|
issue |
|
language |
en
|
license_slug |
ARXIV-1.0
|
number |
|
original_title |
|
pages |
|
publisher |
|
refs |
[]
|
release_date |
2020-06-05
|
release_stage |
submitted
|
release_type |
article
|
release_year |
2020
|
subtitle |
|
title |
Constraining Implicit Space with MDL: Regularity Normalization as Unsupervised Attention
|
version |
v11
|
volume |
|
webcaptures |
[]
|
withdrawn_date |
|
withdrawn_status |
|
withdrawn_year |
|
work_id |
2ulnzgzuyzelrhbls23xmvmyt4
|