@inproceedings{3036025c947d4d6bafb1643ee4cf63f6,
title = "NEURAL COLLAPSE IN DEEP HOMOGENEOUS CLASSIFIERS AND THE ROLE OF WEIGHT DECAY",
abstract = "Neural Collapse is a phenomenon recently discovered in deep classifiers where the last layer activations collapse onto their class means, while the means and last layer weights take on the structure of dual equiangular tight frames. In this paper we present results showing the role of weight decay in the emergence of Neural Collapse in deep homogeneous networks. We show that certain near-interpolating minima of deep networks satisfy the Neural Collapse condition, and this can be derived from the gradient flow on the regularized square loss. We also show that weight decay is necessary for neural collapse to occur. We support our theoretical analysis with experiments that confirm our results.",
keywords = "Classification, Deep Learning, Inductive Bias, Neural Collapse, Weight Decay",
author = "Akshay Rangamani and Andrzej Banburski-Fahey",
note = "Publisher Copyright: {\textcopyright} 2022 IEEE; 47th IEEE International Conference on Acoustics, Speech, and Signal Processing, ICASSP 2022 ; Conference date: 23-05-2022 Through 27-05-2022",
year = "2022",
doi = "10.1109/ICASSP43922.2022.9746778",
language = "English (US)",
series = "ICASSP, IEEE International Conference on Acoustics, Speech and Signal Processing - Proceedings",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "4243--4247",
booktitle = "2022 IEEE International Conference on Acoustics, Speech, and Signal Processing, ICASSP 2022 - Proceedings",
address = "United States",
}