@inproceedings{ma2022effective,
  title={Effective Model Sparsification by Scheduled Grow-and-Prune Methods},
  author={Ma, Xiaolong and Qin, Minghai and Sun, Fei and Hou, Zejiang and Yuan, Kun and Xu, Yi and Wang, Yanzhi and Chen, Yen-Kuang and Jin, Rong and Xie, Yuan},
  booktitle={International Conference on Learning Representations},
  year={2022},
}
@article{jiang2022model,
  title={Model pruning enables efficient federated learning on edge devices},
  author={Jiang, Yuang and Wang, Shiqiang and Valls, Victor and Ko, Bong Jun and Lee, Wei-Han and Leung, Kin K and Tassiulas, Leandros},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  year={2022},
  publisher={IEEE}
}
@inproceedings{wimmer2022interspace,
  title={Interspace Pruning: Using Adaptive Filter Representations to Improve Training of Sparse CNNs},
  author={Wimmer, Paul and Mehnert, Jens and Condurache, Alexandru},
  booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={12527--12537},
  year={2022}
}
@ARTICLE{9451554,  author={Abrahamyan, Lusine and Chen, Yiming and Bekoulis, Giannis and Deligiannis, Nikos},  journal={IEEE Transactions on Neural Networks and Learning Systems},   title={Learned Gradient Compression for Distributed Deep Learning},   year={2021},  volume={},  number={},  pages={1-1},  doi={10.1109/TNNLS.2021.3084806}}
@INPROCEEDINGS{9521328,
  author={Wu, Kun and Jin, Yibo and Miao, Weiwei and Zeng, Zeng and Qian, Zhuzhong and Wang, Jingmian and Zhou, Mingxian and Cao, Tuo},
  booktitle={IEEE/ACM 29th IWQOS}, 
  title={Soudain: Online Adaptive Profile Configuration for Real-time Video Analytics}, 
  year={2021},
  volume={},
  number={},
  pages={1-10},
  doi={10.1109/IWQOS52092.2021.9521328}}
@INPROCEEDINGS{9522274,
  author={Wang, Ning and Duan, Yubin and Wu, Jie},
  booktitle={2021 International Conference on Computer Communications and Networks (ICCCN)}, 
  title={Accelerate Cooperative Deep Inference via Layer-wise Processing Schedule Optimization}, 
  year={2021},
  volume={},
  number={},
  pages={1-9},
  doi={10.1109/ICCCN52240.2021.9522274}}
@inproceedings{10.1145/3472456.3472508,
author = {Yu, Enda and Dong, Dezun and Xu, Yemao and Ouyang, Shuo and Liao, Xiangke},
title = {CD-SGD: Distributed Stochastic Gradient Descent with Compression and Delay Compensation},
year = {2021},
isbn = {9781450390682},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3472456.3472508},
doi = {10.1145/3472456.3472508},
abstract = { Communication overhead is the key challenge for distributed training. Gradient compression is a widely used approach to reduce communication traffic. When combining with a parallel communication mechanism method like pipeline, gradient compression technique can greatly alleviate the impact of communication overhead. However, there exist two problems of gradient compression technique to be solved. Firstly, gradient compression brings in extra computation cost, which will delay the next training iteration. Secondly, gradient compression usually leads to a decrease in convergence accuracy. In this paper, we combine parallel mechanism with gradient quantization and delayed full-gradient compensation, and propose a new distributed optimization method named CD-SGD, which can hide the overhead of gradient compression, overlap part of the communication and obtain high convergence accuracy. The local update operation in CD-SGD allows the next iteration to be launched quickly without waiting for the completion of gradient compression and the current communication process. Besides, the accuracy loss caused by gradient compression is solved by k-step correction method introduced in CD-SGD. We prove that CD-SGD has convergence guarantee and it achieves at least convergence rate. We conduct extensive experiments on MXNet to verify the convergence properties and scaling performance of CD-SGD. Experimental results on a 16-GPU cluster show that convergence accuracy of CD-SGD is close to or even slightly better than that of S-SGD, and its end-to-end time is 30 less than 2-bit gradient compression under a 56Gbps bandwidth environment. },
booktitle = {50th International Conference on Parallel Processing},
articleno = {79},
numpages = {10},
keywords = {Distributed Communication Optimization, Communication Mechanism Optimization, Gradient Compression},
location = {Lemont, IL, USA}
}
@inproceedings{li2021npas,
  title={NPAS: A Compiler-aware Framework of Unified Network Pruning and Architecture Search for Beyond Real-Time Mobile Acceleration},
  author={Li, Zhengang and Yuan, Geng and Niu, Wei and Zhao, Pu and Li, Yanyu and Cai, Yuxuan and Shen, Xuan and Zhan, Zheng and Kong, Zhenglun and Jin, Qing and others},
  booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={14255--14266},
  year={2021}
}
@article{jafari2021fast,
  title={Fast shared-memory streaming multilevel graph partitioning},
  author={Jafari, Nazanin and Selvitopi, Oguz and Aykanat, Cevdet},
  journal={Journal of Parallel and Distributed Computing},
  volume={147},
  pages={140--151},
  year={2021},
  publisher={Elsevier}
}
@article{chang2011libsvm,
  title={LIBSVM: a library for support vector machines},
  author={Chang, Chih-Chung and Lin, Chih-Jen},
  journal={ACM transactions on intelligent systems and technology (TIST)},
  volume={2},
  number={3},
  pages={1--27},
  year={2011},
  publisher={Acm New York, NY, USA}
}
@inproceedings{wang2020joint,
  title={Joint Configuration Adaptation and Bandwidth Allocation for Edge-based Real-time Video Analytics},
  author={Wang, Can and Zhang, Sheng and Chen, Yu and Qian, Zhuzhong and Wu, Jie and Xiao, Mingjun},
  booktitle={IEEE INFOCOM},
  pages={1--10},
  year={2020}
}
@article{taylor2018adaptive,
  title={Adaptive selection of deep learning models on embedded systems},
  author={Taylor, Ben and Marco, Vicent Sanz and Wolff, Willy and Elkhatib, Yehia and Wang, Zheng},
  journal={arXiv preprint arXiv:1805.04252},
  year={2018}
}
@inproceedings{han2016mcdnn,
  title={Mcdnn: An approximation-based execution framework for deep stream processing under resource constraints},
  author={Han, Seungyeop and Shen, Haichen and Philipose, Matthai and Agarwal, Sharad and Wolman, Alec and Krishnamurthy, Arvind},
  booktitle={ACM MobiSys},
  pages={123--136},
  year={2016}
}
@inproceedings{huang2018yolo,
  title={YOLO-LITE: a real-time object detection algorithm optimized for non-GPU computers},
  author={Huang, Rachel and Pedoeem, Jonathan and Chen, Cuixian},
  booktitle={IEEE Big Data},
  pages={2503--2510},
  year={2018},
  organization={IEEE}
}
@article{howard2017mobilenets,
  title={Mobilenets: Efficient convolutional neural networks for mobile vision applications},
  author={Howard, Andrew G and Zhu, Menglong and Chen, Bo and Kalenichenko, Dmitry and Wang, Weijun and Weyand, Tobias and Andreetto, Marco and Adam, Hartwig},
  journal={arXiv preprint arXiv:1704.04861},
  year={2017}
}
@inproceedings{variani2014deep,
  title={Deep neural networks for small footprint text-dependent speaker verification},
  author={Variani, Ehsan and Lei, Xin and McDermott, Erik and Moreno, Ignacio Lopez and Gonzalez-Dominguez, Javier},
  booktitle={IEEE ICASSP},
  pages={4052--4056},
  year={2014},
}
@inproceedings{huang2020deepadapter,
  title={DeepAdapter: A Collaborative Deep Learning Framework for the Mobile Web Using Context-Aware Network Pruning},
  author={Huang, Yakun and Qiao, Xiuquan and Tang, Jian and Ren, Pei and Liu, Ling and Pu, Calton and Chen, Junliang},
  booktitle={IEEE INFOCOM},
  pages={834--843},
  year={2020},
  organization={IEEE}
}
@inproceedings{shi2020communication,
  title={Communication-efficient distributed deep learning with merged gradient sparsification on GPUs},
  author={Shi, Shaohuai and Wang, Qiang and Chu, Xiaowen and Li, Bo and Qin, Yang and Liu, Ruihao and Zhao, Xinxiao},
  booktitle={IEEE INFOCOM},
  pages={406--415},
  year={2020},
  organization={IEEE}
}
@article{m2021efficient,
  title={An Efficient Statistical-based Gradient Compression Technique for Distributed Training Systems},
  author={M Abdelmoniem, Ahmed and Elzanaty, Ahmed and Alouini, Mohamed-Slim and Canini, Marco},
  journal={Machine Learning and Systems},
  volume={3},
  year={2021}
}
@inproceedings{zhang2019compressed,
  title={Compressed distributed gradient descent: Communication-efficient consensus over networks},
  author={Zhang, Xin and Liu, Jia and Zhu, Zhengyuan and Bentley, Elizabeth S},
  booktitle={IEEE INFOCOM},
  pages={2431--2439},
  year={2019},
  organization={IEEE}
}
@article{lin2017deep,
  title={Deep gradient compression: Reducing the communication bandwidth for distributed training},
  author={Lin, Yujun and Han, Song and Mao, Huizi and Wang, Yu and Dally, William J},
  journal={arXiv preprint arXiv:1712.01887},
  year={2017}
}
@inproceedings{chen2018adacomp,
  title={Adacomp: Adaptive residual gradient compression for data-parallel distributed training},
  author={Chen, Chia-Yu and Choi, Jungwook and Brand, Daniel and Agrawal, Ankur and Zhang, Wei and Gopalakrishnan, Kailash},
  booktitle={AAAI Conference on Artificial Intelligence},
  volume={32},
  number={1},
  year={2018}
}
@inproceedings{li2018optimization,
  title={Optimization based Layer-wise Magnitude-based Pruning for DNN Compression.},
  author={Li, Guiying and Qian, Chao and Jiang, Chunhui and Lu, Xiaofen and Tang, Ke},
  booktitle={IJCAI},
  pages={2383--2389},
  year={2018}
}
@inproceedings{niu2020patdnn,
  title={Patdnn: Achieving real-time dnn execution on mobile devices with pattern-based weight pruning},
  author={Niu, Wei and Ma, Xiaolong and Lin, Sheng and Wang, Shihao and Qian, Xuehai and Lin, Xue and Wang, Yanzhi and Ren, Bin},
  booktitle={International Conference on Architectural Support for Programming Languages and Operating Systems},
  pages={907--922},
  year={2020}
}
@inproceedings{ma2020pconv,
  title={Pconv: The missing but desirable sparsity in dnn weight pruning for real-time execution on mobile devices},
  author={Ma, Xiaolong and Guo, Fu-Ming and Niu, Wei and Lin, Xue and Tang, Jian and Ma, Kaisheng and Ren, Bin and Wang, Yanzhi},
  booktitle={AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={5117--5124},
  year={2020}
}
@inproceedings{liu2020autocompress,
  title={AutoCompress: An automatic DNN structured pruning framework for ultra-high compression rates},
  author={Liu, Ning and Ma, Xiaolong and Xu, Zhiyuan and Wang, Yanzhi and Tang, Jian and Ye, Jieping},
  booktitle={AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={4876--4883},
  year={2020}
}
@article{demirci2021partitioning,
  title={Partitioning sparse deep neural networks for scalable training and inference},
  author={Demirci, Gunduz Vehbi and Ferhatosmanoglu, Hakan},
  journal={arXiv preprint arXiv:2104.11805},
  year={2021}
}
@inproceedings{lym2019prunetrain,
  title={PruneTrain: fast neural network training by dynamic sparse model reconfiguration},
  author={Lym, Sangkug and Choukse, Esha and Zangeneh, Siavash and Wen, Wei and Sanghavi, Sujay and Erez, Mattan},
  booktitle={International Conference for High Performance Computing, Networking, Storage and Analysis},
  pages={1--13},
  year={2019}
}
@article{li2019re,
  title={Re-randomized densification for one permutation hashing and bin-wise consistent weighted sampling},
  author={Li, Ping and Li, Xiaoyun and Zhang, Cun Hui},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}
@article{wang2021overlapping,
  title={Overlapping communication with computation in parameter server for scalable dl training},
  author={Wang, Shaoqi and Pi, Aidi and Zhou, Xiaobo and Wang, Jun and Xu, Cheng-Zhong},
  journal={IEEE Transactions on Parallel and Distributed Systems},
  volume={32},
  number={9},
  pages={2144--2159},
  year={2021},
  publisher={IEEE}
}
@article{zhao2020distributed,
  title={Distributed hierarchical gpu parameter server for massive scale deep learning ads systems},
  author={Zhao, Weijie and Xie, Deping and Jia, Ronglai and Qian, Yulei and Ding, Ruiquan and Sun, Mingming and Li, Ping},
  journal={arXiv preprint arXiv:2003.05622},
  year={2020}
}
@inproceedings{pi2020search,
  title={Search-based User Interest Modeling with Lifelong Sequential Behavior Data for Click-Through Rate Prediction},
  author={Pi, Qi and Zhou, Guorui and Zhang, Yujing and Wang, Zhe and Ren, Lejian and Fan, Ying and Zhu, Xiaoqiang and Gai, Kun},
  booktitle={International Conference on Information \& Knowledge Management},
  pages={2685--2692},
  year={2020}
}
@misc{devlin2019bert,
      title={BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding}, 
      author={Jacob Devlin and Ming-Wei Chang and Kenton Lee and Kristina Toutanova},
      year={2019},
      eprint={1810.04805},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}
@inproceedings{zhang2020enabling,
  title={Enabling execution assurance of federated learning at untrusted participants},
  author={Zhang, Xiaoli and Li, Fengting and Zhang, Zeyu and Li, Qi and Wang, Cong and Wu, Jianping},
  booktitle={IEEE INFOCOM},
  pages={1877--1886},
  year={2020},
  organization={IEEE}
}
@article{he2020group,
  title={Group Knowledge Transfer: Federated Learning of Large CNNs at the Edge},
  author={He, Chaoyang and Annavaram, Murali and Avestimehr, Salman},
  journal={NeurIPS},
  volume={33},
  year={2020}
}
@misc{tong2008lessons,
  title={Lessons learned developing a practical large scale machine learning system},
  author={Tong, Simon},
  year={2008}
}
@inproceedings{weinberger2009feature,
  title={Feature hashing for large scale multitask learning},
  author={Weinberger, Kilian and Dasgupta, Anirban and Langford, John and Smola, Alex and Attenberg, Josh},
  booktitle={International Conference on Machine Learning},
  pages={1113--1120},
  year={2009}
}
@article{hoefler2021sparsity,
  title={Sparsity in Deep Learning: Pruning and growth for efficient inference and training in neural networks},
  author={Hoefler, Torsten and Alistarh, Dan and Ben-Nun, Tal and Dryden, Nikoli and Peste, Alexandra},
  journal={arXiv preprint arXiv:2102.00554},
  year={2021}
}
@inproceedings{chen2020elastic,
  title={Elastic parameter server load distribution in deep learning clusters},
  author={Chen, Yangrui and Peng, Yanghua and Bao, Yixin and Wu, Chuan and Zhu, Yibo and Guo, Chuanxiong},
  booktitle={ACM Symposium on Cloud Computing},
  pages={507--521},
  year={2020}
}
@inproceedings{hu2020distributed,
  title={Distributed machine learning through heterogeneous edge systems},
  author={Hu, Hanpeng and Wang, Dan and Wu, Chuan},
  booktitle={AAAI Conference on Artificial Intelligence},
  volume={34},
  number={05},
  pages={7179--7186},
  year={2020}
}
@inproceedings{han2015learning,
  title={Learning both Weights and Connections for Efficient Neural Network},
  author={Han, Song and Pool, Jeff and Tran, John and Dally, William J},
  booktitle={NeurIPS},
  year={2015}
}
@inproceedings{he2020learning,
  title={Learning filter pruning criteria for deep convolutional neural networks acceleration},
  author={He, Yang and Ding, Yuhang and Liu, Ping and Zhu, Linchao and Zhang, Hanwang and Yang, Yi},
  booktitle={IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={2009--2018},
  year={2020}
}
@book{bishop2006pattern,
  title={Pattern recognition and machine learning},
  author={Bishop, Christopher M},
  year={2006},
  publisher={springer}
}
@article{li2019dual,
  title={Dual cross-entropy loss for small-sample fine-grained vehicle classification},
  author={Li, Xiaoxu and Yu, Liyun and Chang, Dongliang and Ma, Zhanyu and Cao, Jie},
  journal={IEEE Transactions on Vehicular Technology},
  volume={68},
  number={5},
  pages={4204--4212},
  year={2019},
  publisher={IEEE}
}
@inproceedings{zhou2018deep,
  title={Deep interest network for click-through rate prediction},
  author={Zhou, Guorui and Zhu, Xiaoqiang and Song, Chenru and Fan, Ying and Zhu, Han and Ma, Xiao and Yan, Yanghui and Jin, Junqi and Li, Han and Gai, Kun},
  booktitle={ACM SIGKDD International Conference on Knowledge Discovery \& Data Mining},
  pages={1059--1068},
  year={2018}
}
@article{duan2021minimizing,
  title={Minimizing Training Time of Distributed Machine Learning by Reducing Data Communication},
  author={Duan, Yubin and Wang, Ning and Wu, Jie},
  journal={IEEE Transactions on Network Science and Engineering},
  year={2021},
  publisher={IEEE}
}
@article{li2011hashing,
  title={Hashing algorithms for large-scale learning},
  author={Li, Ping and Shrivastava, Anshumali and Moore, Joshua and Konig, Arnd Christian},
  journal={arXiv preprint arXiv:1106.0967},
  year={2011}
}
@inproceedings{neglia2019role,
  title={The Role of Network Topology for Distributed Machine Learning},
  author={Neglia, Giovanni and Calbi, Gianmarco and Towsley, Don and Vardoyan, Gayane},
  booktitle={IEEE INFOCOM'19},
  pages={2350--2358},
  year={2019},
  organization={IEEE}
}
@inproceedings{dai2019bigdl,
  title={Bigdl: A distributed deep learning framework for big data},
  author={Dai, Jason Jinquan and Wang, Yiheng and Qiu, Xin and Ding, Ding and Zhang, Yao and Wang, Yanzhang and Jia, Xianyan and Zhang, Cherry Li and Wan, Yan and Li, Zhichao and others},
  booktitle={ACM Symposium on Cloud Computing},
  pages={50--60},
  year={2019}
}
@article{jordan2019ray,
  title={Ray: A Distributed Framework for Emerging $\{$AI$\}$ Applications},
  author={Jordan, Michael I},
  year={2019}
}
@inproceedings{moritz2018ray,
  title={Ray: A distributed framework for emerging {AI} applications},
  author={Moritz, Philipp and Nishihara, Robert and Wang, Stephanie and Tumanov, Alexey and Liaw, Richard and Liang, Eric and Elibol, Melih and Yang, Zongheng and Paul, William and Jordan, Michael I and others},
  booktitle={{USENIX} OSDI'18},
  pages={561--577},
  year={2018}
}
@article{ccatalyurek2011patoh,
  title={Patoh (partitioning tool for hypergraphs)},
  author={{\c{C}}ataly{\"u}rek, {\"U}mit and Aykanat, Cevdet},
  journal={Encyclopedia of Parallel Computing},
  pages={1479--1487},
  year={2011},
  publisher={Springer}
}
@inproceedings{fiduccia1982linear,
  title={A linear-time heuristic for improving network partitions},
  author={Fiduccia, Charles M and Mattheyses, Robert M},
  booktitle={IEEE Design Automation Conference (DAC)},
  pages={175--181},
  year={1982},
  }
  
@article{hendrickson1995multi,
  title={A Multi-Level Algorithm For Partitioning Graphs.},
  author={Hendrickson, Bruce and Leland, Robert W},
  journal={SC},
  volume={95},
  number={28},
  pages={1--14},
  year={1995}
}
@phdthesis{li2017scaling,
  title={Scaling Distributed Machine Learning with System and Algorithm Co-design},
  author={Li, Mu},
  year={2017},
  school={PhD thesis, Intel}
}
@article{meng2016mllib,
  title={Mllib: Machine learning in apache spark},
  author={Meng, Xiangrui and Bradley, Joseph and Yavuz, Burak and Sparks, Evan and Venkataraman, Shivaram and Liu, Davies and Freeman, Jeremy and Tsai, DB and Amde, Manish and Owen, Sean and others},
  journal={The Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={1235--1241},
  year={2016},
  publisher={JMLR.org}
}
@article{low2012distributed,
  title={Distributed GraphLab: a framework for machine learning and data mining in the cloud},
  author={Low, Yucheng and Bickson, Danny and Gonzalez, Joseph and Guestrin, Carlos and Kyrola, Aapo and Hellerstein, Joseph M},
  journal={The Very Large Data Bases Endowment},
  volume={5},
  number={8},
  pages={716--727},
  year={2012},
  publisher={VLDB Endowment}
}
@article{abadi2016tensorflow,
  title={Tensorflow: Large-scale machine learning on heterogeneous distributed systems},
  author={Abadi, Mart{\'\i}n and Agarwal, Ashish and Barham, Paul and Brevdo, Eugene and Chen, Zhifeng and Citro, Craig and Corrado, Greg S and Davis, Andy and Dean, Jeffrey and Devin, Matthieu and others},
  journal={arXiv preprint arXiv:1603.04467},
  year={2016}
}
@book{wu2017distributed,
  title={Distributed {S}ystem {D}esign},
  author={Wu, Jie},
  year={1998},
  publisher={CRC Press}
}
@article{svitkina2011submodular,
  title={Submodular approximation: Sampling-based algorithms and lower bounds},
  author={Svitkina, Zoya and Fleischer, Lisa},
  journal={SIAM Journal on Computing},
  volume={40},
  number={6},
  pages={1715--1737},
  year={2011},
  publisher={SIAM}
}
@inproceedings{wei2015mixed,
  title={Mixed robust/average submodular partitioning: Fast algorithms, guarantees, and applications},
  author={Wei, Kai and Iyer, Rishabh K and Wang, Shengjie and Bai, Wenruo and Bilmes, Jeff A},
  booktitle={NeurIPS},
  pages={2233--2241},
  year={2015}
}

@inproceedings{karypis1996parallel,
  title={Parallel multilevel graph partitioning},
  author={Karypis, George and Kumar, Vipin},
  booktitle={IEEE International Conference on Parallel Processing},
  pages={314--319},
  year={1996},
}
@article{kernighan1970efficient,
  title={An efficient heuristic procedure for partitioning graphs},
  author={Kernighan, Brian W and Lin, Shen},
  journal={Bell system technical journal},
  volume={49},
  number={2},
  pages={291--307},
  year={1970},
  publisher={Wiley Online Library}
}
@inproceedings{krauthgamer2009partitioning,
  title={Partitioning graphs into balanced components},
  author={Krauthgamer, Robert and Naor, Joseph and Schwartz, Roy},
  booktitle={ACM-SIAM Symposium on Discrete Algorithms (SODA)},
  pages={942--949},
  year={2009},
  organization={SIAM}
}
@article{andreev2006balanced,
  title={Balanced graph partitioning},
  author={Andreev, Konstantin and Racke, Harald},
  journal={Theory of Computing Systems},
  volume={39},
  number={6},
  pages={929--939},
  year={2006},
  publisher={Springer}
}
@article{CC01a,
 author = {Chang, Chih-Chung and Lin, Chih-Jen},
 title = {{LIBSVM}: A library for support vector machines},
 journal = {ACM Transactions on Intelligent Systems and Technology},
 volume = {2},
 issue = {3},
 year = {2011},
 pages = {27:1--27:27},
 note =	 {Software available at \url{http://www.csie.ntu.edu.tw/~cjlin/libsvm}}
}
@misc{recordio,
    title = "Create a Dataset Using RecordIO",
    year = "2019",
    url = "https://mxnet.incubator.apache.org/versions/master/faq/recordio.html",
  }
@article{blei2003latent,
  title={Latent dirichlet allocation},
  author={Blei, David M and Ng, Andrew Y and Jordan, Michael I},
  journal={Journal of machine Learning research},
  volume={3},
  number={Jan},
  pages={993--1022},
  year={2003}
}
@article{chen2015mxnet,
  title={Mxnet: A flexible and efficient machine learning library for heterogeneous distributed systems},
  author={Chen, Tianqi and Li, Mu and Li, Yutian and Lin, Min and Wang, Naiyan and Wang, Minjie and Xiao, Tianjun and Xu, Bing and Zhang, Chiyuan and Zhang, Zheng},
  journal={arXiv preprint arXiv:1512.01274},
  year={2015}
}
@article{hastie2005elements,
  title={The elements of statistical learning: data mining, inference and prediction},
  author={Hastie, Trevor and Tibshirani, Robert and Friedman, Jerome and Franklin, James},
  journal={The Mathematical Intelligencer},
  volume={27},
  number={2},
  pages={83--85},
  year={2005},
  publisher={Springer}
}
@misc{wiki:ddr4,
    author = "{Wikipedia contributors}",
    title = "DDR4 SDRAM --- {Wikipedia}{,} The Free Encyclopedia",
    year = "2019",
    url = "https://en.wikipedia.org/w/index.php?title=DDR4_SDRAM&oldid=905603742",
    note = "[Online; accessed 15-July-2019]"
  }
@inproceedings{lasalle2016parallel,
  title={A parallel hill-climbing refinement algorithm for graph partitioning},
  author={LaSalle, Dominique and Karypis, George},
  booktitle={IEEE International Conference on Parallel Processing},
  pages={236--241},
  year={2016},
}
@misc{wiki-pcie,
    author = "{Wikipedia contributors}",
    title = "PCI Express --- {Wikipedia}{,} The Free Encyclopedia",
    year = "2019",
    url = "https://en.wikipedia.org/w/index.php?title=PCI_Express&oldid=905941315",
    note = "[Online; accessed 15-July-2019]"
  }
@inproceedings{wen2017terngrad,
  title={Terngrad: Ternary gradients to reduce communication in distributed deep learning},
  author={Wen, Wei and Xu, Cong and Yan, Feng and Wu, Chunpeng and Wang, Yandan and Chen, Yiran and Li, Hai},
  booktitle={NeurIPS},
  pages={1509--1519},
  year={2017}
}
@inproceedings{shi2019mg,
  title={MG-WFBP: Efficient data communication for distributed synchronous sgd algorithms},
  author={Shi, Shaohuai and Chu, Xiaowen and Li, Bo},
  booktitle={IEEE INFOCOM},
  pages={172--180},
  year={2019},
}
@inproceedings{chen2019round,
  title={Round-robin synchronization: Mitigating communication bottlenecks in parameter servers},
  author={Chen, Chen and Wang, Wei and Li, Bo},
  booktitle={IEEE INFOCOM},
  pages={532--540},
  year={2019},
}
@inproceedings{zhao2019dynamic,
  title={Dynamic Stale Synchronous Parallel Distributed Training for Deep Learning},
  author={Zhao, Xing and An, Aijun and Liu, Junfeng and Chen, Bao Xin},
  booktitle={IEEE ICDCS},
  pages={1507--1517},
  year={2019},
}
@inproceedings{li2014communication,
  title={Communication efficient distributed machine learning with the parameter server},
  author={Li, Mu and Andersen, David G and Smola, Alexander J and Yu, Kai},
  booktitle={NeurIPS},
  pages={19--27},
  year={2014}
}
@inproceedings{bourse2014balanced,
  title={Balanced graph edge partition},
  author={Bourse, Florian and Lelarge, Marc and Vojnovic, Milan},
  booktitle={ACM International Conference on Knowledge Discovery and Data Mining (SIGKDD)},
  pages={1456--1465},
  year={2014},
}
@inproceedings{gonzalez2012powergraph,
  title={Powergraph: Distributed graph-parallel computation on natural graphs},
  author={Gonzalez, Joseph E and Low, Yucheng and Gu, Haijie and Bickson, Danny and Guestrin, Carlos},
  booktitle={USENIX OSDI},
  pages={17--30},
  year={2012}
}
@article{lin2017deep,
  title={Deep gradient compression: Reducing the communication bandwidth for distributed training},
  author={Lin, Yujun and Han, Song and Mao, Huizi and Wang, Yu and Dally, William J},
  journal={arXiv preprint arXiv:1712.01887},
  year={2017}
}
@inproceedings{devine2006parallel,
  title={Parallel hypergraph partitioning for scientific computing},
  author={Devine, Karen D and Boman, Erik G and Heaphy, Robert T and Bisseling, Rob H and Catalyurek, Umit V},
  booktitle={IEEE International Parallel \& Distributed Processing Symposium},
  pages={10--pp},
  year={2006},
}

@article{catalyurek1999hypergraph,
  title={Hypergraph-partitioning-based decomposition for parallel sparse-matrix vector multiplication},
  author={Catalyurek, Umit V and Aykanat, Cevdet},
  journal={IEEE Transactions on parallel and distributed systems},
  volume={10},
  number={7},
  pages={673--693},
  year={1999},
  publisher={IEEE}
}
@article{karypis1998multilevelk,
  title={Multilevelk-way partitioning scheme for irregular graphs},
  author={Karypis, George and Kumar, Vipin},
  journal={Journal of Parallel and Distributed computing},
  volume={48},
  number={1},
  pages={96--129},
  year={1998},
  publisher={Elsevier}
}
@inproceedings{ho2013more,
  title={More effective distributed ml via a stale synchronous parallel parameter server},
  author={Ho, Qirong and Cipar, James and Cui, Henggang and Lee, Seunghak and Kim, Jin Kyu and Gibbons, Phillip B and Gibson, Garth A and Ganger, Greg and Xing, Eric P},
  booktitle={NeurIPS},
  pages={1223--1231},
  year={2013}
}
@inproceedings{li2013parameter,
  title={Parameter server for distributed machine learning},
  author={Li, Mu and Zhou, Li and Yang, Zichao and Li, Aaron and Xia, Fei and Andersen, David G and Smola, Alexander},
  booktitle={NeurIPS Big Learning Workshop},
  pages={2},
  year={2013}
}
@article{smola2010architecture,
  title={An architecture for parallel topic models},
  author={Smola, Alexander and Narayanamurthy, Shravan},
  journal={VLDB Endowment},
  volume={3},
  number={1-2},
  pages={703--710},
  year={2010},
  publisher={VLDB Endowment}
}
@inproceedings{dean2012large,
  title={Large scale distributed deep networks},
  author={Dean, Jeffrey and Corrado, Greg and Monga, Rajat and Chen, Kai and Devin, Matthieu and Mao, Mark and Senior, Andrew and Tucker, Paul and Yang, Ke and Le, Quoc V and others},
  booktitle={NeurIPS},
  pages={1223--1231},
  year={2012}
}
@inproceedings{gonzalez2014graphx,
  title={Graphx: Graph processing in a distributed dataflow framework},
  author={Gonzalez, Joseph E and Xin, Reynold S and Dave, Ankur and Crankshaw, Daniel and Franklin, Michael J and Stoica, Ion},
  booktitle={USENIX OSDI},
  pages={599--613},
  year={2014}
}
@inproceedings{ahmed2012scalable,
  title={Scalable inference in latent variable models},
  author={Ahmed, Amr and Aly, Moahmed and Gonzalez, Joseph and Narayanamurthy, Shravan and Smola, Alexander J},
  booktitle={ACM International Conference on Web Search and Data Mining (WSDM)},
  pages={123--132},
  year={2012},
}
@inproceedings{li2014scaling,
  title={Scaling distributed machine learning with the parameter server},
  author={Li, Mu and Andersen, David G and Park, Jun Woo and Smola, Alexander J and Ahmed, Amr and Josifovski, Vanja and Long, James and Shekita, Eugene J and Su, Bor-Yiing},
  booktitle={USENIX OSDI},
  pages={583--598},
  year={2014}
}
@article{li2015graph,
  title={Graph partitioning via parallel submodular approximation to accelerate distributed machine learning},
  author={Li, Mu and Andersen, Dave G and Smola, Alexander J},
  journal={arXiv preprint arXiv:1505.04636},
  year={2015}
}
@article{Xie2016,
  title={Aggregated Residual Transformations for Deep Neural Networks},
  author={Saining Xie and Ross Girshick and Piotr Dollár and Zhuowen Tu and Kaiming He},
  journal={arXiv preprint arXiv:1611.05431},
  year={2016}
}
@inproceedings{imagenet_cvpr09,
        AUTHOR = {Deng, J. and Dong, W. and Socher, R. and Li, L.-J. and Li, K. and Fei-Fei, L.},
        TITLE = {{ImageNet: A Large-Scale Hierarchical Image Database}},
        BOOKTITLE = { IEEE CVPR},
        YEAR = {2009},
        BIBSOURCE = "http://www.image-net.org/papers/imagenet_cvpr09.bib"}
        
@inproceedings{stanton2012streaming,
  title={Streaming graph partitioning for large distributed graphs},
  author={Stanton, Isabelle and Kliot, Gabriel},
  booktitle={ACM  International Conference on Knowledge Discovery and Data Mining (SIGKDD)},
  pages={1222--1230},
  year={2012},
}
@article{odena2016faster,
  title={Faster asynchronous sgd},
  author={Odena, Augustus},
  journal={arXiv preprint arXiv:1601.04033},
  year={2016}
}