@article{lin_2019, 
  title={Unsupervised Attention Mechanism across Neural Network Layers}, 
  abstractNote={Inspired by the adaptation phenomenon of neuronal firing, we propose an
unsupervised attention mechanism (UAM) which computes the statistical
regularity in the implicit space of neural networks under the Minimum
Description Length (MDL) principle. Treating the neural network optimization
process as a partially observable model selection problem, UAM constrained the
implicit space by a normalization factor, the universal code length. We compute
this universal code incrementally across neural network layers and demonstrated
the flexibility to include data priors such as top-down attention and other
oracle information. Empirically, our approach outperforms existing
normalization methods in tackling limited, imbalanced and nonstationary input
distribution in computer vision and reinforcement learning tasks. Lastly, UAM
tracks dependency and critical learning stages across layers and recurrent time
steps of deep networks.}, 
  author={Lin}, 
  year={2019}, 
  month={Jul}
  }